From ecad5c54cb5d620efb9e938520ca6b19128ad8a3 Mon Sep 17 00:00:00 2001 From: peacewong Date: Mon, 13 Mar 2023 21:18:53 +0800 Subject: [PATCH 001/261] Optimize task planner only create job and codeLogical Task --- .../planner/TaskPlannerTransform.scala | 30 +++++++++++++++---- 1 file changed, 24 insertions(+), 6 deletions(-) diff --git a/linkis-orchestrator/linkis-computation-orchestrator/src/main/scala/org/apache/linkis/orchestrator/computation/catalyst/planner/TaskPlannerTransform.scala b/linkis-orchestrator/linkis-computation-orchestrator/src/main/scala/org/apache/linkis/orchestrator/computation/catalyst/planner/TaskPlannerTransform.scala index f5fc6d5500..69b84fc29d 100644 --- a/linkis-orchestrator/linkis-computation-orchestrator/src/main/scala/org/apache/linkis/orchestrator/computation/catalyst/planner/TaskPlannerTransform.scala +++ b/linkis-orchestrator/linkis-computation-orchestrator/src/main/scala/org/apache/linkis/orchestrator/computation/catalyst/planner/TaskPlannerTransform.scala @@ -34,6 +34,7 @@ import scala.collection.mutable.ArrayBuffer */ class TaskPlannerTransform extends PlannerTransform with Logging { + @Deprecated def rebuildTreeNode(tmpTask: Task): Task = { tmpTask.getChildren.foreach(child => { val newParents = child.getParents.clone() :+ tmpTask @@ -42,6 +43,7 @@ class TaskPlannerTransform extends PlannerTransform with Logging { tmpTask } + @Deprecated def buildCodeLogicTaskTree( codeLogicalUnit: CodeLogicalUnit = null, stage: Stage, @@ -54,6 +56,7 @@ class TaskPlannerTransform extends PlannerTransform with Logging { (rebuildTreeNode(codeLogicalUnitTaskTmp), newStartJobTask) } + @Deprecated def buildStageTaskTree(taskDesc: StageTaskDesc, startJobTask: Task = null): (Task, Task) = { taskDesc match { case endStageTask: EndStageTaskDesc => @@ -102,6 +105,7 @@ class TaskPlannerTransform extends PlannerTransform with Logging { } } + @Deprecated def buildAllStageTaskTree( stages: Array[Stage], startJobTask: Task = null @@ -117,6 +121,7 @@ class TaskPlannerTransform extends PlannerTransform with Logging { (stageTasks.toArray, reusedStartJobTask) } + @Deprecated def buildJobTaskTree(taskDesc: TaskDesc): Task = { taskDesc match { case startTask: StartJobTaskDesc => @@ -133,21 +138,34 @@ class TaskPlannerTransform extends PlannerTransform with Logging { } } + + override def apply(in: Job, context: ASTContext): Task = { in match { case job: CodeJob => - // TODO rebuild needed: Notice( Stages maybe have dependency relation.) - // TODO This class should be split into two kind of transforms. - // TODO First, two PlannerTransforms are needed: one to transform Job to JobTaskEnd, one to transform Job to StageTaskEnd. - // TODO Second, AnalyzeTransforms are needed: one for adding a computationTask by stage for no computation strategy, - // one to transform Job to JobTaskStart, one to transform Job to StageTaskStart. - buildJobTaskTree(EndJobTaskDesc(job)) + val taskDesc = EndJobTaskDesc(job) + val jobTaskTmp = + new JobTask(Array(), Array(buildCodeLogicTaskTree(job.getCodeLogicalUnit, job))) + jobTaskTmp.setTaskDesc(taskDesc) + rebuildNewTreeNode(jobTaskTmp) case _ => logger.error(s"unknown job type:${in.getClass} ") null } } + def rebuildNewTreeNode(tmpTask: Task): Task = { + tmpTask.getChildren.foreach(_.withNewParents(Array(tmpTask))) + tmpTask + } + + def buildCodeLogicTaskTree(codeLogicalUnit: CodeLogicalUnit, job: Job): Task = { + val codeLogicalUnitTaskTmp = new CodeLogicalUnitTask(Array(), Array()) + codeLogicalUnitTaskTmp.setTaskDesc(CodeLogicalUnitTaskDesc(job)) + if (codeLogicalUnit != null) codeLogicalUnitTaskTmp.setCodeLogicalUnit(codeLogicalUnit) + codeLogicalUnitTaskTmp + } + override def getName: String = { val className = getClass.getName if (className endsWith "$") className.dropRight(1) else className From 89dd89a74461ce77387c04697a772d5a0cb40115 Mon Sep 17 00:00:00 2001 From: peacewong Date: Tue, 14 Mar 2023 20:56:02 +0800 Subject: [PATCH 002/261] Add Job ID information to the log of the entire link of the task --- .../governance/common/utils/LoggerUtils.java | 40 +++++++++++ .../AbstractEngineConnLaunchService.scala | 14 +++- .../executor/async/AsyncEngineConnJob.scala | 10 +++ .../executor/async/AsyncExecuteExecutor.scala | 16 +++-- .../execute/ComputationExecutor.scala | 7 +- .../service/TaskExecutionServiceImpl.scala | 14 ++-- .../linkis/entrance/EntranceServer.scala | 6 ++ .../execute/DefaultEntranceExecutor.scala | 5 ++ .../entrance/execute/EntranceExecutor.scala | 14 +++- .../execute/EntranceExecutorManager.scala | 2 +- .../DefaultEngineAskEngineService.scala | 66 ++++++++++++------- .../service/engine/EngineCreateService.scala | 1 - linkis-dist/package/conf/log4j2.xml | 2 +- .../src/main/resources/log4j2.xml | 8 +-- .../flink/src/main/resources/log4j2.xml | 6 +- .../hive/src/main/resources/log4j2.xml | 8 +-- .../io_file/src/main/resources/log4j2.xml | 6 +- .../jdbc/src/main/resources/log4j2.xml | 6 +- .../openlookeng/src/main/resources/log4j2.xml | 6 +- .../pipeline/src/main/resources/log4j2.xml | 6 +- .../presto/src/main/resources/log4j2.xml | 8 +-- .../python/src/main/resources/log4j2.xml | 6 +- .../seatunnel/src/main/resources/log4j2.xml | 6 +- .../shell/src/main/resources/conf/log4j2.xml | 6 +- .../spark/src/main/resources/log4j2.xml | 10 +-- .../sqoop/src/main/resources/log4j2.xml | 8 +-- .../trino/src/main/resources/log4j2.xml | 6 +- .../planner/TaskPlannerTransform.scala | 2 - .../ComputationTaskExecutionReceiver.scala | 13 +++- .../async/AsyncExecTaskRunnerImpl.scala | 50 +++++++------- .../utils/OrchestratorLoggerUtils.scala | 37 +++++++++++ linkis-orchestrator/pom.xml | 1 + 32 files changed, 275 insertions(+), 121 deletions(-) create mode 100644 linkis-computation-governance/linkis-computation-governance-common/src/main/java/org/apache/linkis/governance/common/utils/LoggerUtils.java create mode 100644 linkis-orchestrator/linkis-orchestrator-core/src/main/scala/org/apache/linkis/orchestrator/utils/OrchestratorLoggerUtils.scala diff --git a/linkis-computation-governance/linkis-computation-governance-common/src/main/java/org/apache/linkis/governance/common/utils/LoggerUtils.java b/linkis-computation-governance/linkis-computation-governance-common/src/main/java/org/apache/linkis/governance/common/utils/LoggerUtils.java new file mode 100644 index 0000000000..8ac40a16f4 --- /dev/null +++ b/linkis-computation-governance/linkis-computation-governance-common/src/main/java/org/apache/linkis/governance/common/utils/LoggerUtils.java @@ -0,0 +1,40 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.governance.common.utils; + +import org.apache.linkis.governance.common.constant.job.JobRequestConstants; + +import java.util.Map; + +import org.slf4j.MDC; + +public class LoggerUtils { + + public static void setJobIdMDC(String jobId) { + MDC.put(JobRequestConstants.JOB_ID(), jobId); + } + + public static void setJobIdMDC(Map props) { + String jobId = JobUtils.getJobIdFromMap(props); + MDC.put(JobRequestConstants.JOB_ID(), jobId); + } + + public static void removeJobIdMDC() { + MDC.remove(JobRequestConstants.JOB_ID()); + } +} diff --git a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/service/impl/AbstractEngineConnLaunchService.scala b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/service/impl/AbstractEngineConnLaunchService.scala index 6ac10d1e14..c6c598a2bb 100644 --- a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/service/impl/AbstractEngineConnLaunchService.scala +++ b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/service/impl/AbstractEngineConnLaunchService.scala @@ -29,7 +29,7 @@ import org.apache.linkis.ecm.server.listener.{EngineConnAddEvent, EngineConnStat import org.apache.linkis.ecm.server.service.{EngineConnLaunchService, ResourceLocalizationService} import org.apache.linkis.ecm.server.util.ECMUtils import org.apache.linkis.governance.common.conf.GovernanceCommonConf -import org.apache.linkis.governance.common.utils.JobUtils +import org.apache.linkis.governance.common.utils.{JobUtils, LoggerUtils} import org.apache.linkis.manager.common.entity.enumeration.NodeStatus import org.apache.linkis.manager.common.entity.enumeration.NodeStatus.Failed import org.apache.linkis.manager.common.entity.node.{AMEngineNode, EngineNode} @@ -63,6 +63,7 @@ abstract class AbstractEngineConnLaunchService extends EngineConnLaunchService w override def launchEngineConn(request: EngineConnLaunchRequest, duration: Long): EngineNode = { // 1.创建engineConn和runner,launch 并设置基础属性 val taskId = JobUtils.getJobIdFromStringMap(request.creationDesc.properties) + LoggerUtils.setJobIdMDC(taskId) logger.info("TaskId: {} try to launch a new EngineConn with {}.", taskId: Any, request: Any) val conn = createEngineConn val runner = createEngineConnLaunchRunner @@ -97,15 +98,19 @@ abstract class AbstractEngineConnLaunchService extends EngineConnLaunchService w afterLaunch(request, conn, duration) val future = Future { + LoggerUtils.setJobIdMDC(taskId) logger.info( "TaskId: {} with request {} wait engineConn {} start", Array(taskId, request, conn.getServiceInstance): _* ) - waitEngineConnStart(request, conn, duration) + Utils.tryFinally(waitEngineConnStart(request, conn, duration)) { + LoggerUtils.removeJobIdMDC() + } } future onComplete { case Failure(t) => + LoggerUtils.setJobIdMDC(taskId) logger.error( "TaskId: {} init {} failed. {} with request {}", Array( @@ -120,7 +125,9 @@ abstract class AbstractEngineConnLaunchService extends EngineConnLaunchService w LinkisECMApplication.getContext.getECMSyncListenerBus.postToAll( EngineConnStatusChangeEvent(conn.getTickedId, Failed) ) + LoggerUtils.removeJobIdMDC() case Success(_) => + LoggerUtils.setJobIdMDC(taskId) logger.info( "TaskId: {} init {} succeed. {} with request {}", Array( @@ -132,6 +139,7 @@ abstract class AbstractEngineConnLaunchService extends EngineConnLaunchService w request ): _* ) + LoggerUtils.removeJobIdMDC() } } { t => logger.error( @@ -160,8 +168,10 @@ abstract class AbstractEngineConnLaunchService extends EngineConnLaunchService w LinkisECMApplication.getContext.getECMSyncListenerBus.postToAll( EngineConnStatusChangeEvent(conn.getTickedId, Failed) ) + LoggerUtils.removeJobIdMDC() throw t } + LoggerUtils.removeJobIdMDC() val engineNode = new AMEngineNode() engineNode.setLabels(conn.getLabels) diff --git a/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/async/AsyncEngineConnJob.scala b/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/async/AsyncEngineConnJob.scala index 8876a50c37..6f73f67fe6 100644 --- a/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/async/AsyncEngineConnJob.scala +++ b/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/async/AsyncEngineConnJob.scala @@ -20,6 +20,7 @@ package org.apache.linkis.engineconn.computation.executor.async import org.apache.linkis.common.utils.Utils import org.apache.linkis.engineconn.computation.executor.entity.EngineConnTask import org.apache.linkis.engineconn.computation.executor.execute.EngineExecutionContext +import org.apache.linkis.governance.common.utils.{JobUtils, LoggerUtils} import org.apache.linkis.scheduler.executer.{ CompletedExecuteResponse, ErrorExecuteResponse, @@ -27,6 +28,7 @@ import org.apache.linkis.scheduler.executer.{ SuccessExecuteResponse } import org.apache.linkis.scheduler.queue.{Job, JobInfo} +import org.apache.linkis.scheduler.queue.SchedulerEventState.SchedulerEventState class AsyncEngineConnJob(task: EngineConnTask, engineExecutionContext: EngineExecutionContext) extends Job { @@ -47,6 +49,14 @@ class AsyncEngineConnJob(task: EngineConnTask, engineExecutionContext: EngineExe override def close(): Unit = {} + override def transition(state: SchedulerEventState): Unit = Utils.tryFinally { + val jobId = JobUtils.getJobIdFromMap(task.getProperties) + LoggerUtils.setJobIdMDC(jobId) + super.transition(state) + } { + LoggerUtils.removeJobIdMDC() + } + override def transitionCompleted(executeCompleted: CompletedExecuteResponse): Unit = { var executeCompletedNew: CompletedExecuteResponse = executeCompleted executeCompleted match { diff --git a/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/async/AsyncExecuteExecutor.scala b/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/async/AsyncExecuteExecutor.scala index a27d3f029a..46332b93fd 100644 --- a/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/async/AsyncExecuteExecutor.scala +++ b/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/async/AsyncExecuteExecutor.scala @@ -17,8 +17,10 @@ package org.apache.linkis.engineconn.computation.executor.async +import org.apache.linkis.common.utils.Utils import org.apache.linkis.engineconn.common.exception.EngineConnException import org.apache.linkis.engineconn.computation.executor.utlis.ComputationErrorCode +import org.apache.linkis.governance.common.utils.{JobUtils, LoggerUtils} import org.apache.linkis.scheduler.executer._ import org.apache.linkis.scheduler.executer.ExecutorState.ExecutorState @@ -31,10 +33,16 @@ class AsyncExecuteExecutor(executor: AsyncConcurrentComputationExecutor) extends override def execute(executeRequest: ExecuteRequest): ExecuteResponse = { executeRequest match { case asyncExecuteRequest: AsyncExecuteRequest => - executor.asyncExecuteTask( - asyncExecuteRequest.task, - asyncExecuteRequest.engineExecutionContext - ) + Utils.tryFinally { + val jobId = JobUtils.getJobIdFromMap(asyncExecuteRequest.task.getProperties) + LoggerUtils.setJobIdMDC(jobId) + executor.asyncExecuteTask( + asyncExecuteRequest.task, + asyncExecuteRequest.engineExecutionContext + ) + } { + LoggerUtils.removeJobIdMDC() + } case _ => throw EngineConnException( ComputationErrorCode.ASYNC_EXECUTOR_ERROR_CODE, diff --git a/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/execute/ComputationExecutor.scala b/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/execute/ComputationExecutor.scala index 780db4215a..4571e75039 100644 --- a/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/execute/ComputationExecutor.scala +++ b/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/execute/ComputationExecutor.scala @@ -35,6 +35,7 @@ import org.apache.linkis.engineconn.executor.listener.ExecutorListenerBusContext import org.apache.linkis.governance.common.entity.ExecutionNodeStatus import org.apache.linkis.governance.common.paser.CodeParser import org.apache.linkis.governance.common.protocol.task.{EngineConcurrentInfo, RequestTask} +import org.apache.linkis.governance.common.utils.{JobUtils, LoggerUtils} import org.apache.linkis.manager.common.entity.enumeration.NodeStatus import org.apache.linkis.manager.label.entity.engine.UserCreatorLabel import org.apache.linkis.protocol.engine.JobProgressInfo @@ -257,7 +258,9 @@ abstract class ComputationExecutor(val outputPrintLimit: Int = 1000) } } - def execute(engineConnTask: EngineConnTask): ExecuteResponse = { + def execute(engineConnTask: EngineConnTask): ExecuteResponse = Utils.tryFinally { + val jobId = JobUtils.getJobIdFromMap(engineConnTask.getProperties) + LoggerUtils.setJobIdMDC(jobId) logger.info(s"start to execute task ${engineConnTask.getTaskId}") updateLastActivityTime() beforeExecute(engineConnTask) @@ -271,6 +274,8 @@ abstract class ComputationExecutor(val outputPrintLimit: Int = 1000) logger.info(s"Finished to execute task ${engineConnTask.getTaskId}") // lastTask = null response + } { + LoggerUtils.removeJobIdMDC() } def setCodeParser(codeParser: CodeParser): Unit = this.codeParser = Some(codeParser) diff --git a/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/service/TaskExecutionServiceImpl.scala b/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/service/TaskExecutionServiceImpl.scala index 651fc0f3dc..2b165da9ab 100644 --- a/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/service/TaskExecutionServiceImpl.scala +++ b/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/service/TaskExecutionServiceImpl.scala @@ -58,7 +58,7 @@ import org.apache.linkis.governance.common.exception.engineconn.{ EngineConnExecutorErrorException } import org.apache.linkis.governance.common.protocol.task._ -import org.apache.linkis.governance.common.utils.JobUtils +import org.apache.linkis.governance.common.utils.{JobUtils, LoggerUtils} import org.apache.linkis.manager.common.entity.enumeration.NodeStatus import org.apache.linkis.manager.common.protocol.resource.{ ResponseTaskRunningInfo, @@ -171,7 +171,8 @@ class TaskExecutionServiceImpl @Receiver override def execute(requestTask: RequestTask, sender: Sender): ExecuteResponse = { - + val jobId = JobUtils.getJobIdFromMap(requestTask.getProperties) + LoggerUtils.setJobIdMDC(jobId) // check lock logger.info("Received a new task, task content is " + requestTask) if (StringUtils.isBlank(requestTask.getLock)) { @@ -208,7 +209,7 @@ class TaskExecutionServiceImpl if (null != retry) retry.asInstanceOf[Boolean] else false } - val jobId = JobUtils.getJobIdFromMap(requestTask.getProperties) + if (StringUtils.isNotBlank(jobId)) { System.getProperties.put(ComputationExecutorConf.JOB_ID_TO_ENV_KEY, jobId) logger.info(s"Received job with id ${jobId}.") @@ -227,6 +228,7 @@ class TaskExecutionServiceImpl override def run(): Unit = Utils.tryCatch { // Waiting to run, preventing task messages from being sent to submit services before SubmitResponse, such as entry Thread.sleep(ComputationExecutorConf.TASK_SUBMIT_WAIT_TIME_MS) + LoggerUtils.setJobIdMDC(jobId) submitTaskToExecutor(task, labels) match { case ErrorExecuteResponse(message, throwable) => sendToEntrance(task, ResponseTaskError(task.getTaskId, message)) @@ -256,11 +258,11 @@ class TaskExecutionServiceImpl case computationExecutor: ComputationExecutor => taskIdCache.put(task.getTaskId, computationExecutor) submitTask(task, computationExecutor) - case o => + case _ => val labelsStr = if (labels != null) labels.filter(_ != null).map(_.getStringValue).mkString(",") else "" val msg = - "Invalid computationExecutor : " + o.getClass.getName + ", labels : " + labelsStr + ", requestTask : " + task.getTaskId + "Invalid computationExecutor : " + executor.getClass.getName + ", labels : " + labelsStr + ", requestTask : " + task.getTaskId logger.error(msg) ErrorExecuteResponse( "Invalid computationExecutor(生成无效的计算引擎，请联系管理员).", @@ -389,6 +391,8 @@ class TaskExecutionServiceImpl } private def executeTask(task: EngineConnTask, executor: ComputationExecutor): Unit = { + val jobId = JobUtils.getJobIdFromMap(task.getProperties) + LoggerUtils.setJobIdMDC(jobId) val response = executor.execute(task) response match { case ErrorExecuteResponse(message, throwable) => diff --git a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/EntranceServer.scala b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/EntranceServer.scala index f298e54251..1035de1e2b 100644 --- a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/EntranceServer.scala +++ b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/EntranceServer.scala @@ -28,6 +28,7 @@ import org.apache.linkis.entrance.log.LogReader import org.apache.linkis.entrance.timeout.JobTimeoutManager import org.apache.linkis.entrance.utils.JobHistoryHelper import org.apache.linkis.governance.common.entity.job.JobRequest +import org.apache.linkis.governance.common.utils.LoggerUtils import org.apache.linkis.protocol.constants.TaskConstant import org.apache.linkis.rpc.Sender import org.apache.linkis.scheduler.queue.{Job, SchedulerEventState} @@ -78,12 +79,15 @@ abstract class EntranceServer extends Logging { } logger.info(s"received a request,convert $jobRequest") + LoggerUtils.setJobIdMDC(jobRequest.getId.toString) + val logAppender = new java.lang.StringBuilder() Utils.tryThrow( getEntranceContext .getOrCreateEntranceInterceptors() .foreach(int => jobRequest = int.apply(jobRequest, logAppender)) ) { t => + LoggerUtils.removeJobIdMDC() val error = t match { case error: ErrorException => error case t1: Throwable => @@ -165,8 +169,10 @@ abstract class EntranceServer extends Logging { entranceJob.getLogListener.foreach(_.onLogUpdate(entranceJob, msg)) case _ => } + LoggerUtils.removeJobIdMDC() job } { t => + LoggerUtils.removeJobIdMDC() job.onFailure("Submitting the query failed!(提交查询失败！)", t) val _jobRequest: JobRequest = getEntranceContext.getOrCreateEntranceParser().parseToJobRequest(job) diff --git a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/execute/DefaultEntranceExecutor.scala b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/execute/DefaultEntranceExecutor.scala index c509f10056..266de6eb5b 100644 --- a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/execute/DefaultEntranceExecutor.scala +++ b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/execute/DefaultEntranceExecutor.scala @@ -25,6 +25,7 @@ import org.apache.linkis.entrance.orchestrator.EntranceOrchestrationFactory import org.apache.linkis.entrance.utils.JobHistoryHelper import org.apache.linkis.governance.common.entity.ExecutionNodeStatus import org.apache.linkis.governance.common.protocol.task.ResponseTaskStatus +import org.apache.linkis.governance.common.utils.LoggerUtils import org.apache.linkis.manager.label.entity.Label import org.apache.linkis.manager.label.entity.engine.CodeLanguageLabel import org.apache.linkis.manager.label.utils.LabelUtil @@ -106,6 +107,7 @@ class DefaultEntranceExecutor(id: Long) entranceExecuteRequest: EntranceExecuteRequest, orchestration: Orchestration ): Unit = { + LoggerUtils.setJobIdMDC(getId.toString) orchestrationResponse match { case succeedResponse: SucceedTaskResponse => succeedResponse match { @@ -184,6 +186,7 @@ class DefaultEntranceExecutor(id: Long) _.onLogUpdate(entranceExecuteRequest.getJob, LogUtils.generateERROR(msg)) ) } + LoggerUtils.removeJobIdMDC() } def requestToComputationJobReq(entranceExecuteRequest: EntranceExecuteRequest): JobReq = { @@ -238,11 +241,13 @@ class DefaultEntranceExecutor(id: Long) } override def kill(): Boolean = { + LoggerUtils.setJobIdMDC(getId.toString) logger.info("Entrance start to kill job {} invoke Orchestrator ", this.getId) Utils.tryAndWarn { val msg = s"You job with id was cancelled by user!" getRunningOrchestrationFuture.foreach(_.cancel(msg)) } + LoggerUtils.removeJobIdMDC() true } diff --git a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/execute/EntranceExecutor.scala b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/execute/EntranceExecutor.scala index 44cb3620ce..be7fb13871 100644 --- a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/execute/EntranceExecutor.scala +++ b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/execute/EntranceExecutor.scala @@ -19,9 +19,11 @@ package org.apache.linkis.entrance.execute import org.apache.linkis.common.log.LogUtils import org.apache.linkis.common.utils.{Logging, Utils} +import org.apache.linkis.entrance.exception.{EntranceErrorCode, EntranceErrorException} import org.apache.linkis.entrance.job.EntranceExecuteRequest import org.apache.linkis.governance.common.entity.ExecutionNodeStatus._ import org.apache.linkis.governance.common.protocol.task.{RequestTask, ResponseTaskStatus} +import org.apache.linkis.governance.common.utils.LoggerUtils import org.apache.linkis.orchestrator.computation.operation.log.LogProcessor import org.apache.linkis.orchestrator.computation.operation.progress.ProgressProcessor import org.apache.linkis.orchestrator.computation.operation.resource.ResourceReportProcessor @@ -29,6 +31,7 @@ import org.apache.linkis.orchestrator.core.OrchestrationFuture import org.apache.linkis.protocol.UserWithCreator import org.apache.linkis.scheduler.executer._ import org.apache.linkis.scheduler.executer.ExecutorState.ExecutorState +import org.apache.linkis.server.BDPJettyServerHelper import org.apache.commons.io.IOUtils import org.apache.commons.lang3.StringUtils @@ -69,9 +72,14 @@ abstract class EntranceExecutor(val id: Long) extends Executor with Logging { } override def execute(executeRequest: ExecuteRequest): ExecuteResponse = { - var request: RequestTask = null - interceptors.foreach(in => request = in.apply(request, executeRequest)) - callExecute(executeRequest) + LoggerUtils.setJobIdMDC(getId.toString) + Utils.tryFinally { + var request: RequestTask = null + interceptors.foreach(in => request = in.apply(request, executeRequest)) + callExecute(executeRequest) + } { + LoggerUtils.removeJobIdMDC() + } } protected def callback(): Unit = {} diff --git a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/execute/EntranceExecutorManager.scala b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/execute/EntranceExecutorManager.scala index 0d5d605983..05bc5311b0 100644 --- a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/execute/EntranceExecutorManager.scala +++ b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/execute/EntranceExecutorManager.scala @@ -90,7 +90,7 @@ abstract class EntranceExecutorManager(groupFactory: GroupFactory) job.getJobRequest match { case jobReq: JobRequest => val entranceEntranceExecutor = - new DefaultEntranceExecutor(idGenerator.incrementAndGet()) + new DefaultEntranceExecutor(jobReq.getId) // getEngineConn Executor job.getLogListener.foreach( _.onLogUpdate( diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/service/engine/DefaultEngineAskEngineService.scala b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/service/engine/DefaultEngineAskEngineService.scala index 4616cf85a1..080ae49a7a 100644 --- a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/service/engine/DefaultEngineAskEngineService.scala +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/service/engine/DefaultEngineAskEngineService.scala @@ -19,7 +19,7 @@ package org.apache.linkis.manager.am.service.engine import org.apache.linkis.common.exception.LinkisRetryException import org.apache.linkis.common.utils.{Logging, Utils} -import org.apache.linkis.governance.common.utils.JobUtils +import org.apache.linkis.governance.common.utils.{JobUtils, LoggerUtils} import org.apache.linkis.manager.am.conf.AMConfiguration import org.apache.linkis.manager.common.constant.AMConstant import org.apache.linkis.manager.common.protocol.engine._ @@ -68,6 +68,7 @@ class DefaultEngineAskEngineService @Receiver override def askEngine(engineAskRequest: EngineAskRequest, sender: Sender): Any = { val taskId = JobUtils.getJobIdFromStringMap(engineAskRequest.getProperties) + LoggerUtils.setJobIdMDC(taskId) logger.info(s"received task: $taskId, engineAskRequest $engineAskRequest") if (!engineAskRequest.getLabels.containsKey(LabelKeyConstant.EXECUTE_ONCE_KEY)) { val engineReuseRequest = new EngineReuseRequest() @@ -97,6 +98,7 @@ class DefaultEngineAskEngineService val engineAskAsyncId = getAsyncId val createNodeThread = Future { + LoggerUtils.setJobIdMDC(taskId) logger.info( s"Task: $taskId start to async($engineAskAsyncId) createEngine, ${engineAskRequest.getCreateService}" ) @@ -108,30 +110,40 @@ class DefaultEngineAskEngineService engineCreateRequest.setUser(engineAskRequest.getUser) engineCreateRequest.setProperties(engineAskRequest.getProperties) engineCreateRequest.setCreateService(engineAskRequest.getCreateService) - val createNode = engineCreateService.createEngine(engineCreateRequest, sender) - val timeout = - if (engineCreateRequest.getTimeout <= 0) { - AMConfiguration.ENGINE_START_MAX_TIME.getValue.toLong - } else engineCreateRequest.getTimeout - // useEngine 需要加上超时 - val createEngineNode = getEngineNodeManager.useEngine(createNode, timeout) - if (null == createEngineNode) { - throw new LinkisRetryException( - AMConstant.EM_ERROR_CODE, - s"create engine${createNode.getServiceInstance} success, but to use engine failed" + Utils.tryFinally { + val createNode = engineCreateService.createEngine(engineCreateRequest, sender) + val timeout = + if (engineCreateRequest.getTimeout <= 0) { + AMConfiguration.ENGINE_START_MAX_TIME.getValue.toLong + } else engineCreateRequest.getTimeout + // useEngine 需要加上超时 + val createEngineNode = getEngineNodeManager.useEngine(createNode, timeout) + if (null == createEngineNode) { + throw new LinkisRetryException( + AMConstant.EM_ERROR_CODE, + s"create engine${createNode.getServiceInstance} success, but to use engine failed" + ) + } + logger.info( + s"Task: $taskId finished to ask engine for user ${engineAskRequest.getUser} by create node $createEngineNode" ) + createEngineNode + } { + LoggerUtils.removeJobIdMDC() } - logger.info( - s"Task: $taskId finished to ask engine for user ${engineAskRequest.getUser} by create node $createEngineNode" - ) - createEngineNode } createNodeThread.onComplete { case Success(engineNode) => - logger.info(s"Task: $taskId Success to async($engineAskAsyncId) createEngine $engineNode") - sender.send(EngineCreateSuccess(engineAskAsyncId, engineNode)) + LoggerUtils.setJobIdMDC(taskId) + Utils.tryFinally { + logger.info(s"Task: $taskId Success to async($engineAskAsyncId) createEngine $engineNode") + sender.send(EngineCreateSuccess(engineAskAsyncId, engineNode)) + } { + LoggerUtils.removeJobIdMDC() + } case Failure(exception) => + LoggerUtils.setJobIdMDC(taskId) val retryFlag = exception match { case retryException: LinkisRetryException => true case retryableException: RetryableException => true @@ -151,15 +163,19 @@ class DefaultEngineAskEngineService logger.info(s"msg: ${msg} canRetry Exception: ${exception.getClass.getName}") } - sender.send( - EngineCreateError( - engineAskAsyncId, - ExceptionUtils.getRootCauseMessage(exception), - retryFlag + Utils.tryFinally { + sender.send( + EngineCreateError( + engineAskAsyncId, + ExceptionUtils.getRootCauseMessage(exception), + retryFlag + ) ) - ) + } { + LoggerUtils.removeJobIdMDC() + } } - + LoggerUtils.removeJobIdMDC() EngineAskAsyncResponse(engineAskAsyncId, Sender.getThisServiceInstance) } diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/service/engine/EngineCreateService.scala b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/service/engine/EngineCreateService.scala index 8e90e222c9..c1efd0f964 100644 --- a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/service/engine/EngineCreateService.scala +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/service/engine/EngineCreateService.scala @@ -17,7 +17,6 @@ package org.apache.linkis.manager.am.service.engine -import org.apache.linkis.common.ServiceInstance import org.apache.linkis.common.exception.LinkisRetryException import org.apache.linkis.manager.common.entity.node.EngineNode import org.apache.linkis.manager.common.protocol.engine.EngineCreateRequest diff --git a/linkis-dist/package/conf/log4j2.xml b/linkis-dist/package/conf/log4j2.xml index c10f78e6c7..f25111c3c0 100644 --- a/linkis-dist/package/conf/log4j2.xml +++ b/linkis-dist/package/conf/log4j2.xml @@ -20,7 +20,7 @@ - + diff --git a/linkis-engineconn-plugins/elasticsearch/src/main/resources/log4j2.xml b/linkis-engineconn-plugins/elasticsearch/src/main/resources/log4j2.xml index 020b94567e..79f5a3e5bc 100644 --- a/linkis-engineconn-plugins/elasticsearch/src/main/resources/log4j2.xml +++ b/linkis-engineconn-plugins/elasticsearch/src/main/resources/log4j2.xml @@ -20,23 +20,23 @@ - + - + - + - + diff --git a/linkis-engineconn-plugins/flink/src/main/resources/log4j2.xml b/linkis-engineconn-plugins/flink/src/main/resources/log4j2.xml index 4e2c961ae3..126a2a6cef 100644 --- a/linkis-engineconn-plugins/flink/src/main/resources/log4j2.xml +++ b/linkis-engineconn-plugins/flink/src/main/resources/log4j2.xml @@ -20,7 +20,7 @@ - + @@ -31,12 +31,12 @@ - + - + diff --git a/linkis-engineconn-plugins/hive/src/main/resources/log4j2.xml b/linkis-engineconn-plugins/hive/src/main/resources/log4j2.xml index e6f8b150f3..00a4f181af 100644 --- a/linkis-engineconn-plugins/hive/src/main/resources/log4j2.xml +++ b/linkis-engineconn-plugins/hive/src/main/resources/log4j2.xml @@ -20,7 +20,7 @@ - + @@ -29,18 +29,18 @@ - + - + - + diff --git a/linkis-engineconn-plugins/io_file/src/main/resources/log4j2.xml b/linkis-engineconn-plugins/io_file/src/main/resources/log4j2.xml index 7852867ba3..dbbcf39cba 100644 --- a/linkis-engineconn-plugins/io_file/src/main/resources/log4j2.xml +++ b/linkis-engineconn-plugins/io_file/src/main/resources/log4j2.xml @@ -20,7 +20,7 @@ - + @@ -31,12 +31,12 @@ - + - + diff --git a/linkis-engineconn-plugins/jdbc/src/main/resources/log4j2.xml b/linkis-engineconn-plugins/jdbc/src/main/resources/log4j2.xml index 83186732fc..11d3045e76 100644 --- a/linkis-engineconn-plugins/jdbc/src/main/resources/log4j2.xml +++ b/linkis-engineconn-plugins/jdbc/src/main/resources/log4j2.xml @@ -20,7 +20,7 @@ - + @@ -31,12 +31,12 @@ - + - + diff --git a/linkis-engineconn-plugins/openlookeng/src/main/resources/log4j2.xml b/linkis-engineconn-plugins/openlookeng/src/main/resources/log4j2.xml index e9e7bb70ff..a30b0f91d0 100644 --- a/linkis-engineconn-plugins/openlookeng/src/main/resources/log4j2.xml +++ b/linkis-engineconn-plugins/openlookeng/src/main/resources/log4j2.xml @@ -20,17 +20,17 @@ - + - + - + diff --git a/linkis-engineconn-plugins/pipeline/src/main/resources/log4j2.xml b/linkis-engineconn-plugins/pipeline/src/main/resources/log4j2.xml index 03517a50e8..c0f41df284 100644 --- a/linkis-engineconn-plugins/pipeline/src/main/resources/log4j2.xml +++ b/linkis-engineconn-plugins/pipeline/src/main/resources/log4j2.xml @@ -20,7 +20,7 @@ - + @@ -31,12 +31,12 @@ - + - + diff --git a/linkis-engineconn-plugins/presto/src/main/resources/log4j2.xml b/linkis-engineconn-plugins/presto/src/main/resources/log4j2.xml index 020b94567e..26bedbfde8 100644 --- a/linkis-engineconn-plugins/presto/src/main/resources/log4j2.xml +++ b/linkis-engineconn-plugins/presto/src/main/resources/log4j2.xml @@ -20,23 +20,23 @@ - + - + - + - + diff --git a/linkis-engineconn-plugins/python/src/main/resources/log4j2.xml b/linkis-engineconn-plugins/python/src/main/resources/log4j2.xml index a385c888fa..04f4613525 100644 --- a/linkis-engineconn-plugins/python/src/main/resources/log4j2.xml +++ b/linkis-engineconn-plugins/python/src/main/resources/log4j2.xml @@ -20,7 +20,7 @@ - + @@ -31,12 +31,12 @@ - + - + diff --git a/linkis-engineconn-plugins/seatunnel/src/main/resources/log4j2.xml b/linkis-engineconn-plugins/seatunnel/src/main/resources/log4j2.xml index 83186732fc..11d3045e76 100644 --- a/linkis-engineconn-plugins/seatunnel/src/main/resources/log4j2.xml +++ b/linkis-engineconn-plugins/seatunnel/src/main/resources/log4j2.xml @@ -20,7 +20,7 @@ - + @@ -31,12 +31,12 @@ - + - + diff --git a/linkis-engineconn-plugins/shell/src/main/resources/conf/log4j2.xml b/linkis-engineconn-plugins/shell/src/main/resources/conf/log4j2.xml index 68173f23ac..744ac4908a 100644 --- a/linkis-engineconn-plugins/shell/src/main/resources/conf/log4j2.xml +++ b/linkis-engineconn-plugins/shell/src/main/resources/conf/log4j2.xml @@ -20,7 +20,7 @@ - + @@ -29,14 +29,14 @@ - + - + diff --git a/linkis-engineconn-plugins/spark/src/main/resources/log4j2.xml b/linkis-engineconn-plugins/spark/src/main/resources/log4j2.xml index b02d5889cd..b330214ec8 100644 --- a/linkis-engineconn-plugins/spark/src/main/resources/log4j2.xml +++ b/linkis-engineconn-plugins/spark/src/main/resources/log4j2.xml @@ -20,7 +20,7 @@ - + @@ -29,23 +29,23 @@ - + - + - + - + diff --git a/linkis-engineconn-plugins/sqoop/src/main/resources/log4j2.xml b/linkis-engineconn-plugins/sqoop/src/main/resources/log4j2.xml index 9fd15a4b48..bff135ad07 100644 --- a/linkis-engineconn-plugins/sqoop/src/main/resources/log4j2.xml +++ b/linkis-engineconn-plugins/sqoop/src/main/resources/log4j2.xml @@ -20,23 +20,23 @@ - + - + - + - + diff --git a/linkis-engineconn-plugins/trino/src/main/resources/log4j2.xml b/linkis-engineconn-plugins/trino/src/main/resources/log4j2.xml index 9cf70f16c1..4150900fd8 100644 --- a/linkis-engineconn-plugins/trino/src/main/resources/log4j2.xml +++ b/linkis-engineconn-plugins/trino/src/main/resources/log4j2.xml @@ -20,16 +20,16 @@ - + - + - + diff --git a/linkis-orchestrator/linkis-computation-orchestrator/src/main/scala/org/apache/linkis/orchestrator/computation/catalyst/planner/TaskPlannerTransform.scala b/linkis-orchestrator/linkis-computation-orchestrator/src/main/scala/org/apache/linkis/orchestrator/computation/catalyst/planner/TaskPlannerTransform.scala index 69b84fc29d..ab404dba6f 100644 --- a/linkis-orchestrator/linkis-computation-orchestrator/src/main/scala/org/apache/linkis/orchestrator/computation/catalyst/planner/TaskPlannerTransform.scala +++ b/linkis-orchestrator/linkis-computation-orchestrator/src/main/scala/org/apache/linkis/orchestrator/computation/catalyst/planner/TaskPlannerTransform.scala @@ -138,8 +138,6 @@ class TaskPlannerTransform extends PlannerTransform with Logging { } } - - override def apply(in: Job, context: ASTContext): Task = { in match { case job: CodeJob => diff --git a/linkis-orchestrator/linkis-computation-orchestrator/src/main/scala/org/apache/linkis/orchestrator/computation/service/ComputationTaskExecutionReceiver.scala b/linkis-orchestrator/linkis-computation-orchestrator/src/main/scala/org/apache/linkis/orchestrator/computation/service/ComputationTaskExecutionReceiver.scala index 521205927a..101914724c 100644 --- a/linkis-orchestrator/linkis-computation-orchestrator/src/main/scala/org/apache/linkis/orchestrator/computation/service/ComputationTaskExecutionReceiver.scala +++ b/linkis-orchestrator/linkis-computation-orchestrator/src/main/scala/org/apache/linkis/orchestrator/computation/service/ComputationTaskExecutionReceiver.scala @@ -17,8 +17,7 @@ package org.apache.linkis.orchestrator.computation.service -import org.apache.linkis.common.log.LogUtils -import org.apache.linkis.common.utils.{Logging, Utils} +import org.apache.linkis.common.utils.Logging import org.apache.linkis.governance.common.entity.ExecutionNodeStatus import org.apache.linkis.governance.common.protocol.task._ import org.apache.linkis.manager.common.protocol.resource.ResponseTaskRunningInfo @@ -28,6 +27,7 @@ import org.apache.linkis.orchestrator.computation.monitor.EngineConnMonitor import org.apache.linkis.orchestrator.core.ResultSet import org.apache.linkis.orchestrator.ecm.service.TaskExecutionReceiver import org.apache.linkis.orchestrator.listener.task._ +import org.apache.linkis.orchestrator.utils.OrchestratorLoggerUtils import org.apache.linkis.rpc.Sender import org.apache.linkis.rpc.message.annotation.Receiver import org.apache.linkis.rpc.utils.RPCUtils @@ -91,6 +91,7 @@ class ComputationTaskExecutionReceiver extends TaskExecutionReceiver with Loggin codeExecTaskExecutorManager .getByEngineConnAndTaskId(serviceInstance, taskStatus.execId) .foreach { codeExecutor => + OrchestratorLoggerUtils.setJobIdMDC(codeExecutor.getExecTask) val event = TaskStatusEvent(codeExecutor.getExecTask, taskStatus.status) logger.info( s"From engineConn receive status info:$taskStatus, now post to listenerBus event: $event" @@ -111,6 +112,7 @@ class ComputationTaskExecutionReceiver extends TaskExecutionReceiver with Loggin ) } } + OrchestratorLoggerUtils.removeJobIdMDC() } @Receiver @@ -123,6 +125,7 @@ class ComputationTaskExecutionReceiver extends TaskExecutionReceiver with Loggin codeExecTaskExecutorManager .getByEngineConnAndTaskId(serviceInstance, taskResultSize.execId) .foreach { codeExecutor => + OrchestratorLoggerUtils.setJobIdMDC(codeExecutor.getExecTask) val event = TaskResultSetSizeEvent(codeExecutor.getExecTask, taskResultSize.resultSize) logger.info( s"From engineConn receive resultSet size info$taskResultSize, now post to listenerBus event: $event" @@ -134,6 +137,7 @@ class ComputationTaskExecutionReceiver extends TaskExecutionReceiver with Loggin if (!isExist) { logger.warn(s"from $serviceInstance received $taskResultSize cannot find execTask to deal") } + OrchestratorLoggerUtils.removeJobIdMDC() } @Receiver @@ -143,6 +147,7 @@ class ComputationTaskExecutionReceiver extends TaskExecutionReceiver with Loggin codeExecTaskExecutorManager .getByEngineConnAndTaskId(serviceInstance, taskResultSet.execId) .foreach { codeExecutor => + OrchestratorLoggerUtils.setJobIdMDC(codeExecutor.getExecTask) val event = TaskResultSetEvent( codeExecutor.getExecTask, ResultSet(taskResultSet.output, taskResultSet.alias) @@ -157,6 +162,7 @@ class ComputationTaskExecutionReceiver extends TaskExecutionReceiver with Loggin if (!isExist) { logger.warn(s"from $serviceInstance received $taskResultSet cannot find execTask to deal") } + OrchestratorLoggerUtils.removeJobIdMDC() } @Receiver @@ -166,6 +172,7 @@ class ComputationTaskExecutionReceiver extends TaskExecutionReceiver with Loggin codeExecTaskExecutorManager .getByEngineConnAndTaskId(serviceInstance, responseTaskError.execId) .foreach { codeExecutor => + OrchestratorLoggerUtils.setJobIdMDC(codeExecutor.getExecTask) val event = TaskErrorResponseEvent(codeExecutor.getExecTask, responseTaskError.errorMsg) logger.info( s"From engineConn receive responseTaskError info${responseTaskError.execId}, now post to listenerBus event: ${event.execTask.getIDInfo()}" @@ -177,7 +184,7 @@ class ComputationTaskExecutionReceiver extends TaskExecutionReceiver with Loggin if (!isExist) { logger.warn(s"from $serviceInstance received $responseTaskError cannot find execTask to deal") } - + OrchestratorLoggerUtils.removeJobIdMDC() } } diff --git a/linkis-orchestrator/linkis-orchestrator-core/src/main/scala/org/apache/linkis/orchestrator/strategy/async/AsyncExecTaskRunnerImpl.scala b/linkis-orchestrator/linkis-orchestrator-core/src/main/scala/org/apache/linkis/orchestrator/strategy/async/AsyncExecTaskRunnerImpl.scala index d546889b67..459bf7f38b 100644 --- a/linkis-orchestrator/linkis-orchestrator-core/src/main/scala/org/apache/linkis/orchestrator/strategy/async/AsyncExecTaskRunnerImpl.scala +++ b/linkis-orchestrator/linkis-orchestrator-core/src/main/scala/org/apache/linkis/orchestrator/strategy/async/AsyncExecTaskRunnerImpl.scala @@ -17,20 +17,16 @@ package org.apache.linkis.orchestrator.strategy.async -import org.apache.linkis.common.utils.Logging +import org.apache.linkis.common.utils.{Logging, Utils} import org.apache.linkis.governance.common.entity.ExecutionNodeStatus import org.apache.linkis.orchestrator.conf.OrchestratorConfiguration import org.apache.linkis.orchestrator.core.ResultSet import org.apache.linkis.orchestrator.exception.OrchestratorErrorCodeSummary -import org.apache.linkis.orchestrator.execution.{ArrayResultSetTaskResponse, _} +import org.apache.linkis.orchestrator.execution._ import org.apache.linkis.orchestrator.execution.impl.{ DefaultFailedTaskResponse, DefaultResultSetTaskResponse } -import org.apache.linkis.orchestrator.listener.{ - OrchestratorListenerBusContext, - OrchestratorSyncListenerBus -} import org.apache.linkis.orchestrator.listener.execution.ExecTaskRunnerCompletedEvent import org.apache.linkis.orchestrator.plans.physical.ExecTask import org.apache.linkis.orchestrator.strategy.{ @@ -38,6 +34,7 @@ import org.apache.linkis.orchestrator.strategy.{ ResultSetExecTask, StatusInfoExecTask } +import org.apache.linkis.orchestrator.utils.OrchestratorLoggerUtils import scala.collection.mutable.ArrayBuffer @@ -66,25 +63,26 @@ class AsyncExecTaskRunnerImpl(override val task: ExecTask) override def isSucceed: Boolean = ExecutionNodeStatus.isScheduled(status) - override def run(): Unit = try { - logger.info(s"ExecTaskRunner Submit execTask(${task.getIDInfo}) to running") - val response = task.execute() - this.taskResponse = response - response match { - case async: AsyncTaskResponse => - transientStatus(ExecutionNodeStatus.Running) - case succeed: SucceedTaskResponse => - logger.info(s"Succeed to execute ExecTask(${task.getIDInfo})") - transientStatus(ExecutionNodeStatus.Succeed) - case failedTaskResponse: FailedTaskResponse => - logger.info(s"Failed to execute ExecTask(${task.getIDInfo})") - transientStatus(ExecutionNodeStatus.Failed) - case retry: RetryTaskResponse => - logger.warn(s"ExecTask(${task.getIDInfo}) need to retry") - transientStatus(ExecutionNodeStatus.WaitForRetry) - } - } catch { - case e: Throwable => + override def run(): Unit = { + Utils.tryCatch { + OrchestratorLoggerUtils.setJobIdMDC(task) + logger.info(s"ExecTaskRunner Submit execTask(${task.getIDInfo}) to running") + val response = task.execute() + this.taskResponse = response + response match { + case async: AsyncTaskResponse => + transientStatus(ExecutionNodeStatus.Running) + case succeed: SucceedTaskResponse => + logger.info(s"Succeed to execute ExecTask(${task.getIDInfo})") + transientStatus(ExecutionNodeStatus.Succeed) + case failedTaskResponse: FailedTaskResponse => + logger.info(s"Failed to execute ExecTask(${task.getIDInfo})") + transientStatus(ExecutionNodeStatus.Failed) + case retry: RetryTaskResponse => + logger.warn(s"ExecTask(${task.getIDInfo}) need to retry") + transientStatus(ExecutionNodeStatus.WaitForRetry) + } + } { case e: Throwable => logger.error(s"Failed to execute task ${task.getIDInfo}", e) this.taskResponse = new DefaultFailedTaskResponse( e.getMessage, @@ -92,6 +90,8 @@ class AsyncExecTaskRunnerImpl(override val task: ExecTask) e ) transientStatus(ExecutionNodeStatus.Failed) + } + OrchestratorLoggerUtils.removeJobIdMDC() } override def transientStatus(status: ExecutionNodeStatus): Unit = { diff --git a/linkis-orchestrator/linkis-orchestrator-core/src/main/scala/org/apache/linkis/orchestrator/utils/OrchestratorLoggerUtils.scala b/linkis-orchestrator/linkis-orchestrator-core/src/main/scala/org/apache/linkis/orchestrator/utils/OrchestratorLoggerUtils.scala new file mode 100644 index 0000000000..e249c675bb --- /dev/null +++ b/linkis-orchestrator/linkis-orchestrator-core/src/main/scala/org/apache/linkis/orchestrator/utils/OrchestratorLoggerUtils.scala @@ -0,0 +1,37 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.orchestrator.utils + +import org.apache.linkis.governance.common.utils.LoggerUtils +import org.apache.linkis.orchestrator.plans.physical.ExecTask + +object OrchestratorLoggerUtils { + + def setJobIdMDC(task: ExecTask): Unit = { + val startUpMap = + task.getTaskDesc.getOrigin.getASTOrchestration.getASTContext.getParams.getStartupParams + if (null != startUpMap) { + LoggerUtils.setJobIdMDC(startUpMap.getConfigurationMap()) + } + } + + def removeJobIdMDC(): Unit = { + LoggerUtils.removeJobIdMDC() + } + +} diff --git a/linkis-orchestrator/pom.xml b/linkis-orchestrator/pom.xml index afc94da056..f424c38285 100644 --- a/linkis-orchestrator/pom.xml +++ b/linkis-orchestrator/pom.xml @@ -21,6 +21,7 @@ org.apache.linkis linkis ${revision} + ../pom.xml linkis-orchestrator From a6013bdf46b0273e19b844c6fb9ce3227c9f63dd Mon Sep 17 00:00:00 2001 From: peacewong Date: Wed, 15 Mar 2023 19:47:47 +0800 Subject: [PATCH 003/261] Fix log out cookie not to clear --- .../linkis/gateway/security/GatewaySSOUtils.scala | 12 +++++++++++- .../apache/linkis/gateway/security/UserRestful.scala | 10 +++++++--- 2 files changed, 18 insertions(+), 4 deletions(-) diff --git a/linkis-spring-cloud-services/linkis-service-gateway/linkis-gateway-core/src/main/scala/org/apache/linkis/gateway/security/GatewaySSOUtils.scala b/linkis-spring-cloud-services/linkis-service-gateway/linkis-gateway-core/src/main/scala/org/apache/linkis/gateway/security/GatewaySSOUtils.scala index 675944013e..2d696d8410 100644 --- a/linkis-spring-cloud-services/linkis-service-gateway/linkis-gateway-core/src/main/scala/org/apache/linkis/gateway/security/GatewaySSOUtils.scala +++ b/linkis-spring-cloud-services/linkis-service-gateway/linkis-gateway-core/src/main/scala/org/apache/linkis/gateway/security/GatewaySSOUtils.scala @@ -113,7 +113,17 @@ object GatewaySSOUtils extends Logging { def removeLoginUser(gatewayContext: GatewayContext): Unit = { SSOUtils.removeLoginUser(gatewayContext.getRequest.getCookies.asScala.flatMap(_._2).toArray) - SSOUtils.removeLoginUserByAddCookie(c => gatewayContext.getResponse.addCookie(c)) + SSOUtils.removeLoginUserByAddCookie(c => { + if (cookieDomainSetupSwitch) { + val host = gatewayContext.getRequest.getHeaders.get("Host") + if (host != null && host.nonEmpty) { + c.setDomain( + getCookieDomain(host.head, GatewayConfiguration.GATEWAY_DOMAIN_LEVEL.getValue) + ) + } + } + gatewayContext.getResponse.addCookie(c) + }) } def updateLastAccessTime(gatewayContext: GatewayContext): Unit = diff --git a/linkis-spring-cloud-services/linkis-service-gateway/linkis-gateway-core/src/main/scala/org/apache/linkis/gateway/security/UserRestful.scala b/linkis-spring-cloud-services/linkis-service-gateway/linkis-gateway-core/src/main/scala/org/apache/linkis/gateway/security/UserRestful.scala index 6e3aca6c91..b0f42f9ad0 100644 --- a/linkis-spring-cloud-services/linkis-service-gateway/linkis-gateway-core/src/main/scala/org/apache/linkis/gateway/security/UserRestful.scala +++ b/linkis-spring-cloud-services/linkis-service-gateway/linkis-gateway-core/src/main/scala/org/apache/linkis/gateway/security/UserRestful.scala @@ -238,9 +238,13 @@ abstract class UserPwdAbstractUserRestful extends AbstractUserRestful with Loggi def clearExpireCookie(gatewayContext: GatewayContext): Unit = { val cookies = gatewayContext.getRequest.getCookies.values().asScala.flatMap(cookie => cookie).toArray - val expireCookies = cookies.filter(cookie => - cookie.getName.equals(ServerConfiguration.LINKIS_SERVER_SESSION_TICKETID_KEY.getValue) - ) + val expireCookies = cookies.filter { cookie => + cookie.getName.equals( + ServerConfiguration.LINKIS_SERVER_SESSION_TICKETID_KEY.getValue + ) || cookie.getName.equals( + ServerConfiguration.LINKIS_SERVER_SESSION_PROXY_TICKETID_KEY.getValue + ) + } val host = gatewayContext.getRequest.getHeaders.get("Host") if (host != null && host.nonEmpty) { val maxDomainLevel = host.head.split("\\.").length From 81be80069ae3575c5e14a3e984f41ecfb0781634 Mon Sep 17 00:00:00 2001 From: peacewong Date: Thu, 16 Mar 2023 10:49:39 +0800 Subject: [PATCH 004/261] Task progress exceeding 1 will give fake progress --- .../governance/common/conf/GovernaceCommonConf.scala | 2 ++ .../executor/service/TaskExecutionServiceImpl.scala | 8 +++----- .../entrance/persistence/QueryPersistenceManager.java | 6 ++++++ .../spark/executor/SparkEngineConnExecutor.scala | 4 +++- 4 files changed, 14 insertions(+), 6 deletions(-) diff --git a/linkis-computation-governance/linkis-computation-governance-common/src/main/scala/org/apache/linkis/governance/common/conf/GovernaceCommonConf.scala b/linkis-computation-governance/linkis-computation-governance-common/src/main/scala/org/apache/linkis/governance/common/conf/GovernaceCommonConf.scala index 48f906bb8c..53d0df8f92 100644 --- a/linkis-computation-governance/linkis-computation-governance-common/src/main/scala/org/apache/linkis/governance/common/conf/GovernaceCommonConf.scala +++ b/linkis-computation-governance/linkis-computation-governance-common/src/main/scala/org/apache/linkis/governance/common/conf/GovernaceCommonConf.scala @@ -66,6 +66,8 @@ object GovernanceCommonConf { val ERROR_CODE_DESC_LEN = CommonVars("linkis.error.code.desc.len", 512, "Error code description maximum length").getValue + val FAKE_PROGRESS: Float = CommonVars[Float]("linkis.job.fake.progress", 0.99f).getValue + def getEngineEnvValue(envKey: String): String = { CommonVars(envKey, "").getValue } diff --git a/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/service/TaskExecutionServiceImpl.scala b/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/service/TaskExecutionServiceImpl.scala index 651fc0f3dc..d68cb69608 100644 --- a/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/service/TaskExecutionServiceImpl.scala +++ b/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/service/TaskExecutionServiceImpl.scala @@ -503,12 +503,10 @@ class TaskExecutionServiceImpl if (ExecutionNodeStatus.isCompleted(task.getStatus)) { response = ResponseTaskProgress(taskID, 1.0f, null) } else { + val progress = executor.progress(taskID) + logger.info("The latest progress {} of the task id {} is:", progress, taskID) response = Utils.tryQuietly( - ResponseTaskProgress( - taskID, - executor.progress(taskID), - executor.getProgressInfo(taskID) - ) + ResponseTaskProgress(taskID, progress, executor.getProgressInfo(taskID)) ) } } else { diff --git a/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/persistence/QueryPersistenceManager.java b/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/persistence/QueryPersistenceManager.java index 39964fdad1..44966fc1f1 100644 --- a/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/persistence/QueryPersistenceManager.java +++ b/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/persistence/QueryPersistenceManager.java @@ -23,6 +23,7 @@ import org.apache.linkis.entrance.cs.CSEntranceHelper; import org.apache.linkis.entrance.execute.EntranceJob; import org.apache.linkis.entrance.log.FlexibleErrorCodeManager; +import org.apache.linkis.governance.common.conf.GovernanceCommonConf; import org.apache.linkis.governance.common.entity.job.JobRequest; import org.apache.linkis.protocol.engine.JobProgressInfo; import org.apache.linkis.scheduler.executer.OutputExecuteResponse; @@ -169,6 +170,11 @@ private void updateJobStatus(Job job) { JobRequest jobRequest = null; if (job.isCompleted()) { job.setProgress(1); + } else if (job.getProgress() >= 1 && job instanceof EntranceJob) { + job.setProgress(GovernanceCommonConf.FAKE_PROGRESS()); + ((EntranceJob) job) + .getJobRequest() + .setProgress(String.valueOf(GovernanceCommonConf.FAKE_PROGRESS())); } try { jobRequest = this.entranceContext.getOrCreateEntranceParser().parseToJobRequest(job); diff --git a/linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/executor/SparkEngineConnExecutor.scala b/linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/executor/SparkEngineConnExecutor.scala index f2a40dd06b..cb2f6187ed 100644 --- a/linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/executor/SparkEngineConnExecutor.scala +++ b/linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/executor/SparkEngineConnExecutor.scala @@ -33,6 +33,7 @@ import org.apache.linkis.engineplugin.spark.extension.{ SparkPreExecutionHook } import org.apache.linkis.engineplugin.spark.utils.JobProgressUtil +import org.apache.linkis.governance.common.conf.GovernanceCommonConf import org.apache.linkis.governance.common.exception.LinkisJobRetryException import org.apache.linkis.governance.common.utils.JobUtils import org.apache.linkis.manager.common.entity.enumeration.NodeStatus @@ -169,7 +170,8 @@ abstract class SparkEngineConnExecutor(val sc: SparkContext, id: Long) val newProgress = (engineExecutionContext.getCurrentParagraph * 1f - 1f) / engineExecutionContext.getTotalParagraph + JobProgressUtil .progress(sc, jobGroup) / engineExecutionContext.getTotalParagraph - val normalizedProgress = if (newProgress >= 1) newProgress - 0.1f else newProgress + val normalizedProgress = + if (newProgress >= 1) GovernanceCommonConf.FAKE_PROGRESS else newProgress val oldProgress = ProgressUtils.getOldProgress(this.engineExecutionContext) if (normalizedProgress < oldProgress) oldProgress else { From 7809b007815217ba2fb9fa7a9a7107fdb233f346 Mon Sep 17 00:00:00 2001 From: davidhua Date: Mon, 7 Nov 2022 02:20:10 +0800 Subject: [PATCH 005/261] Enable to custom the storage path for yarn logs in ECM. --- .../operator/EngineConnLogOperator.scala | 12 +- .../operator/EngineConnYarnLogOperator.scala | 181 ++++++++++++------ 2 files changed, 133 insertions(+), 60 deletions(-) diff --git a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/operator/EngineConnLogOperator.scala b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/operator/EngineConnLogOperator.scala index bc856ba681..b558c319cd 100644 --- a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/operator/EngineConnLogOperator.scala +++ b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/operator/EngineConnLogOperator.scala @@ -95,10 +95,7 @@ class EngineConnLogOperator extends Operator with Logging { var readLine, skippedLine, lineNum = 0 var rowIgnore = false var ignoreLine = 0 - val linePattern = Option(EngineConnLogOperator.MULTILINE_PATTERN.getValue) match { - case Some(pattern) => pattern.r - case _ => null - } + val linePattern = getLinePattern val maxMultiline = EngineConnLogOperator.MULTILINE_MAX.getValue Utils.tryFinally { var line = randomAndReversedReadLine() @@ -155,6 +152,13 @@ class EngineConnLogOperator extends Operator with Logging { logPath } + protected def getLinePattern: Regex = { + Option(EngineConnLogOperator.MULTILINE_PATTERN.getValue) match { + case Some(pattern) => pattern.r + case _ => null + } + } + protected def getEngineConnInfo(implicit parameters: Map[String, Any] ): (String, String, String) = { diff --git a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/operator/EngineConnYarnLogOperator.scala b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/operator/EngineConnYarnLogOperator.scala index e00d16b519..1dc51be3a1 100644 --- a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/operator/EngineConnYarnLogOperator.scala +++ b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/operator/EngineConnYarnLogOperator.scala @@ -17,20 +17,35 @@ package org.apache.linkis.ecm.server.operator -import org.apache.linkis.common.exception.LinkisCommonErrorException +import org.apache.linkis.common.conf.CommonVars +import org.apache.linkis.common.io.FsPath import org.apache.linkis.common.utils.Utils import org.apache.linkis.ecm.core.conf.ECMErrorCode -import org.apache.linkis.ecm.errorcode.EngineconnServerErrorCodeSummary._ import org.apache.linkis.ecm.server.exception.ECMErrorException +import org.apache.linkis.storage.FSFactory +import org.apache.linkis.storage.fs.FileSystem +import org.apache.linkis.storage.utils.StorageUtils + +import org.apache.commons.lang3.StringUtils import java.io.File -import java.text.MessageFormat -import java.util.concurrent.TimeUnit +import java.util +import java.util.concurrent.{Callable, ConcurrentHashMap, ExecutorService, Future, TimeUnit} import scala.collection.JavaConverters._ +import scala.util.matching.Regex class EngineConnYarnLogOperator extends EngineConnLogOperator { + private implicit val fs: FileSystem = + FSFactory.getFs(StorageUtils.FILE).asInstanceOf[FileSystem] + + /** + * Yarn log fetchers + */ + private def yarnLogFetchers: ConcurrentHashMap[String, Future[String]] = + new ConcurrentHashMap[String, Future[String]]() + override def getNames: Array[String] = Array(EngineConnYarnLogOperator.OPERATOR_NAME) override def apply(implicit parameters: Map[String, Any]): Map[String, Any] = { @@ -38,75 +53,118 @@ class EngineConnYarnLogOperator extends EngineConnLogOperator { Utils.tryFinally { result = super.apply(parameters) result - } { - result.get("logPath") match { - case Some(path: String) => - val logFile = new File(path) - if (logFile.exists() && logFile.getName.startsWith(".")) { - // If is a temporary file, drop it - logger.info(s"Delete the temporary yarn log file: [$path]") - if (!logFile.delete()) { - logger.warn(s"Fail to delete the temporary yarn log file: [$path]") - } - } - } - } + } {} } override def getLogPath(implicit parameters: Map[String, Any]): File = { + val applicationId = getAsThrow[String]("yarnApplicationId") val (ticketId, engineConnInstance, engineConnLogDir) = getEngineConnInfo(parameters) - val rootLogDir = new File(engineConnLogDir) - if (!rootLogDir.exists() || !rootLogDir.isDirectory) { + val rootLogPath = EngineConnYarnLogOperator.YARN_LOG_STORAGE_PATH.getValue match { + case storePath if StringUtils.isNotBlank(storePath) => + val logPath = new FsPath(StorageUtils.FILE_SCHEMA + storePath + "/" + applicationId) + // Invoke to create directory + fs.mkdir(logPath) + // 777 permission + fs.setPermission(logPath, "rwxrwxrwx") + logPath + case _ => new FsPath(StorageUtils.FILE_SCHEMA + engineConnLogDir) + } + if (!fs.exists(rootLogPath) || !rootLogPath.toFile.isDirectory) { throw new ECMErrorException( - LOG_IS_NOT_EXISTS.getErrorCode, - MessageFormat.format(LOG_IS_NOT_EXISTS.getErrorDesc, rootLogDir) + ECMErrorCode.EC_FETCH_LOG_FAILED, + s"Log directory ${rootLogPath.getPath} is not exists." ) } val creator = getAsThrow[String]("creator") - val applicationId = getAsThrow[String]("yarnApplicationId") - var logPath = new File(engineConnLogDir, "yarn_" + applicationId) - if (!logPath.exists()) { - val tempLogFile = - s".yarn_${applicationId}_${System.currentTimeMillis()}_${Thread.currentThread().getId}" - Utils.tryCatch { - var command = s"yarn logs -applicationId $applicationId >> $rootLogDir/$tempLogFile" - logger.info(s"Fetch yarn logs to temporary file: [$command]") - val processBuilder = new ProcessBuilder(sudoCommands(creator, command): _*) - processBuilder.environment.putAll(sys.env.asJava) - processBuilder.redirectErrorStream(false) - val process = processBuilder.start() - val waitFor = process.waitFor(5, TimeUnit.SECONDS) - logger.trace(s"waitFor: ${waitFor}, result: ${process.exitValue()}") - if (waitFor && process.waitFor() == 0) { - command = s"mv $rootLogDir/$tempLogFile $rootLogDir/yarn_$applicationId" - logger.info(s"Move and save yarn logs: [$command]") - Utils.exec(sudoCommands(creator, command)) - } else { - logPath = new File(engineConnLogDir, tempLogFile) - if (!logPath.exists()) { - throw new LinkisCommonErrorException( - -1, - s"Fetch yarn logs timeout, log aggregation has not completed or is not enabled" - ) - } + var logPath = new FsPath( + StorageUtils.FILE_SCHEMA + rootLogPath.getPath + "/yarn_" + applicationId + ) + if (!fs.exists(logPath)) { + val fetcher = yarnLogFetchers.computeIfAbsent( + applicationId, + new util.function.Function[String, Future[String]] { + override def apply(v1: String): Future[String] = + requestToFetchYarnLogs(creator, applicationId, rootLogPath.getPath) } - } { case e: Exception => - throw new LinkisCommonErrorException( - -1, - s"Fail to fetch yarn logs application: $applicationId, message: ${e.getMessage}" - ) + ) + // Just wait 5 seconds + Option(fetcher.get(5, TimeUnit.SECONDS)) match { + case Some(path) => logPath = new FsPath(StorageUtils.FILE_SCHEMA + path) + case _ => } + } - if (!logPath.exists() || !logPath.isFile) { + if (!fs.exists(logPath) || logPath.toFile.isDirectory) { throw new ECMErrorException( - LOGFILE_IS_NOT_EXISTS.getErrorCode, - MessageFormat.format(LOGFILE_IS_NOT_EXISTS.getErrorDesc, logPath) + ECMErrorCode.EC_FETCH_LOG_FAILED, + s"LogFile $logPath is not exists or is not a file." ) } logger.info( s"Try to fetch EngineConn(id: $ticketId, instance: $engineConnInstance) yarn logs from ${logPath.getPath} in application id: $applicationId" ) - logPath + logPath.toFile + } + + /** + * Not support line pattern in yarn log + * @return + */ + override protected def getLinePattern: Regex = null + + /** + * Request the log fetcher + * + * @param creator + * creator + * @param applicationId + * application id + * @param logPath + * log path + * @return + */ + private def requestToFetchYarnLogs( + creator: String, + applicationId: String, + yarnLogDir: String + ): Future[String] = { + EngineConnYarnLogOperator.YARN_LOG_FETCH_SCHEDULER.submit(new Callable[String] { + override def call(): String = { + val logPath = new FsPath(StorageUtils.FILE_SCHEMA + yarnLogDir + "/yarn_" + applicationId) + if (!fs.exists(logPath)) { + val tempLogFile = + s".yarn_${applicationId}_${System.currentTimeMillis()}_${Thread.currentThread().getId}" + Utils.tryCatch { + var command = + s"yarn logs -applicationId $applicationId >> $yarnLogDir/$tempLogFile" + logger.info(s"Fetch yarn logs to temporary file: [$command]") + val processBuilder = new ProcessBuilder(sudoCommands(creator, command): _*) + processBuilder.environment.putAll(sys.env.asJava) + processBuilder.redirectErrorStream(false) + val process = processBuilder.start() + val exitCode = process.waitFor() + logger.trace(s"Finish to fetch yan logs to temporary file, result: ${exitCode}") + if (exitCode == 0) { + command = s"mv $yarnLogDir/$tempLogFile $yarnLogDir/yarn_$applicationId" + logger.info(s"Move and save yarn logs(${applicationId}): [$command]") + Utils.exec(sudoCommands(creator, command)) + } + } { e: Throwable => + logger.error( + s"Fail to fetch yarn logs application: $applicationId, message: ${e.getMessage}" + ) + } + val tmpFile = new File(yarnLogDir, tempLogFile) + if (tmpFile.exists()) { + logger.info(s"Delete temporary file: [${tempLogFile}] in yarn logs fetcher") + tmpFile.delete() + } + } + // Remove future + yarnLogFetchers.remove(applicationId) + if (fs.exists(logPath)) logPath.getPath else null + } + }) } private def sudoCommands(creator: String, command: String): Array[String] = { @@ -121,4 +179,15 @@ class EngineConnYarnLogOperator extends EngineConnLogOperator { object EngineConnYarnLogOperator { val OPERATOR_NAME = "engineConnYarnLog" + + // Specific the path to store the yarn logs + val YARN_LOG_STORAGE_PATH: CommonVars[String] = + CommonVars("linkis.engineconn.log.yarn.storage-path", "") + + val YARN_LOG_FETCH_THREAD: CommonVars[Int] = + CommonVars("linkis.engineconn.log.yarn.fetch.thread-num", 5) + + val YARN_LOG_FETCH_SCHEDULER: ExecutorService = + Utils.newFixedThreadPool(YARN_LOG_FETCH_THREAD.getValue + 1, "yarn_logs_fetch", false) + } From 1ca18b702d2bec223fe6aabf5213fac1548e7833 Mon Sep 17 00:00:00 2001 From: davidhua Date: Thu, 27 Oct 2022 17:06:09 +0800 Subject: [PATCH 006/261] Change the default value of blank-placeholder in Flink engine; --- .../engineconnplugin/flink/config/FlinkEnvConfiguration.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/linkis-engineconn-plugins/flink/src/main/scala/org/apache/linkis/engineconnplugin/flink/config/FlinkEnvConfiguration.scala b/linkis-engineconn-plugins/flink/src/main/scala/org/apache/linkis/engineconnplugin/flink/config/FlinkEnvConfiguration.scala index 466dab95ed..6bdb228191 100644 --- a/linkis-engineconn-plugins/flink/src/main/scala/org/apache/linkis/engineconnplugin/flink/config/FlinkEnvConfiguration.scala +++ b/linkis-engineconn-plugins/flink/src/main/scala/org/apache/linkis/engineconnplugin/flink/config/FlinkEnvConfiguration.scala @@ -118,6 +118,6 @@ object FlinkEnvConfiguration { val FLINK_KERBEROS_CONF_PATH = CommonVars("linkis.flink.kerberos.krb5-conf.path", "") val FLINK_PARAMS_BLANK_PLACEHOLER = - CommonVars("linkis.flink.params.placeholder.blank", "\\0x001") + CommonVars("linkis.flink.params.placeholder.blank", "\u0001") } From 4f8551c8f6c854f4214f612c93d8fa9e3c4ec7e3 Mon Sep 17 00:00:00 2001 From: peacewong Date: Thu, 16 Mar 2023 15:27:06 +0800 Subject: [PATCH 007/261] optimize errorcode --- .../operator/EngineConnYarnLogOperator.scala | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/operator/EngineConnYarnLogOperator.scala b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/operator/EngineConnYarnLogOperator.scala index 1dc51be3a1..36e7ddfc5f 100644 --- a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/operator/EngineConnYarnLogOperator.scala +++ b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/operator/EngineConnYarnLogOperator.scala @@ -20,7 +20,7 @@ package org.apache.linkis.ecm.server.operator import org.apache.linkis.common.conf.CommonVars import org.apache.linkis.common.io.FsPath import org.apache.linkis.common.utils.Utils -import org.apache.linkis.ecm.core.conf.ECMErrorCode +import org.apache.linkis.ecm.errorcode.EngineconnServerErrorCodeSummary._ import org.apache.linkis.ecm.server.exception.ECMErrorException import org.apache.linkis.storage.FSFactory import org.apache.linkis.storage.fs.FileSystem @@ -29,6 +29,7 @@ import org.apache.linkis.storage.utils.StorageUtils import org.apache.commons.lang3.StringUtils import java.io.File +import java.text.MessageFormat import java.util import java.util.concurrent.{Callable, ConcurrentHashMap, ExecutorService, Future, TimeUnit} @@ -71,8 +72,8 @@ class EngineConnYarnLogOperator extends EngineConnLogOperator { } if (!fs.exists(rootLogPath) || !rootLogPath.toFile.isDirectory) { throw new ECMErrorException( - ECMErrorCode.EC_FETCH_LOG_FAILED, - s"Log directory ${rootLogPath.getPath} is not exists." + LOG_IS_NOT_EXISTS.getErrorCode, + MessageFormat.format(LOG_IS_NOT_EXISTS.getErrorDesc, rootLogPath.getPath) ) } val creator = getAsThrow[String]("creator") @@ -96,8 +97,8 @@ class EngineConnYarnLogOperator extends EngineConnLogOperator { } if (!fs.exists(logPath) || logPath.toFile.isDirectory) { throw new ECMErrorException( - ECMErrorCode.EC_FETCH_LOG_FAILED, - s"LogFile $logPath is not exists or is not a file." + LOGFILE_IS_NOT_EXISTS.getErrorCode, + MessageFormat.format(LOGFILE_IS_NOT_EXISTS.getErrorDesc, logPath.getPath) ) } logger.info( @@ -124,10 +125,10 @@ class EngineConnYarnLogOperator extends EngineConnLogOperator { * @return */ private def requestToFetchYarnLogs( - creator: String, - applicationId: String, - yarnLogDir: String - ): Future[String] = { + creator: String, + applicationId: String, + yarnLogDir: String + ): Future[String] = { EngineConnYarnLogOperator.YARN_LOG_FETCH_SCHEDULER.submit(new Callable[String] { override def call(): String = { val logPath = new FsPath(StorageUtils.FILE_SCHEMA + yarnLogDir + "/yarn_" + applicationId) From 83d7f0e501a672a8b135e20f5586460707511915 Mon Sep 17 00:00:00 2001 From: davidhua Date: Tue, 15 Nov 2022 20:17:34 +0800 Subject: [PATCH 008/261] Clean the job resources after submitting of flink-jar job in flink engine. --- .../flink/executor/FlinkOnceExecutor.scala | 16 +++- .../FlinkJobResCleanInterceptor.scala | 89 +++++++++++++++++++ .../FlinkJobSubmitInterceptor.scala | 50 +++++++++++ .../FlinkApplicationExecutorFactory.scala | 12 ++- .../FlinkJobLocalResourceCleaner.scala | 70 +++++++++++++++ .../resource/FlinkJobResourceCleaner.scala | 38 ++++++++ 6 files changed, 273 insertions(+), 2 deletions(-) create mode 100644 linkis-engineconn-plugins/flink/src/main/scala/org/apache/linkis/engineconnplugin/flink/executor/interceptor/FlinkJobResCleanInterceptor.scala create mode 100644 linkis-engineconn-plugins/flink/src/main/scala/org/apache/linkis/engineconnplugin/flink/executor/interceptor/FlinkJobSubmitInterceptor.scala create mode 100644 linkis-engineconn-plugins/flink/src/main/scala/org/apache/linkis/engineconnplugin/flink/resource/FlinkJobLocalResourceCleaner.scala create mode 100644 linkis-engineconn-plugins/flink/src/main/scala/org/apache/linkis/engineconnplugin/flink/resource/FlinkJobResourceCleaner.scala diff --git a/linkis-engineconn-plugins/flink/src/main/scala/org/apache/linkis/engineconnplugin/flink/executor/FlinkOnceExecutor.scala b/linkis-engineconn-plugins/flink/src/main/scala/org/apache/linkis/engineconnplugin/flink/executor/FlinkOnceExecutor.scala index 411f574e7f..f5edf97535 100644 --- a/linkis-engineconn-plugins/flink/src/main/scala/org/apache/linkis/engineconnplugin/flink/executor/FlinkOnceExecutor.scala +++ b/linkis-engineconn-plugins/flink/src/main/scala/org/apache/linkis/engineconnplugin/flink/executor/FlinkOnceExecutor.scala @@ -32,6 +32,7 @@ import org.apache.linkis.engineconnplugin.flink.config.FlinkEnvConfiguration.{ } import org.apache.linkis.engineconnplugin.flink.errorcode.FlinkErrorCodeSummary._ import org.apache.linkis.engineconnplugin.flink.exception.ExecutorInitException +import org.apache.linkis.engineconnplugin.flink.executor.interceptor.FlinkJobSubmitInterceptor import org.apache.linkis.manager.common.entity.enumeration.NodeStatus import org.apache.flink.api.common.JobStatus @@ -47,6 +48,8 @@ trait FlinkOnceExecutor[T <: ClusterDescriptorAdapter] protected var clusterDescriptor: T = _ private var daemonThread: Future[_] = _ + private var interceptor: FlinkJobSubmitInterceptor = _ + protected def submit(onceExecutorExecutionContext: OnceExecutorExecutionContext): Unit = { ClusterDescriptorAdapterFactory.create(flinkEngineConnContext.getExecutionContext) match { case adapter: T => clusterDescriptor = adapter @@ -58,7 +61,14 @@ trait FlinkOnceExecutor[T <: ClusterDescriptorAdapter] case (k, v) if v != null => k -> v.toString case (k, _) => k -> null }.toMap - doSubmit(onceExecutorExecutionContext, options) + Option(interceptor).foreach(op => op.beforeSubmit(onceExecutorExecutionContext)) + Utils.tryCatch { + doSubmit(onceExecutorExecutionContext, options) + Option(interceptor).foreach(op => op.afterSubmitSuccess(onceExecutorExecutionContext)) + } { t: Throwable => + Option(interceptor).foreach(op => op.afterSubmitFail(onceExecutorExecutionContext, t)) + throw t + } if (isCompleted) return if (null == clusterDescriptor.getClusterID) { throw new ExecutorInitException(YARN_IS_NULL.getErrorDesc) @@ -82,6 +92,10 @@ trait FlinkOnceExecutor[T <: ClusterDescriptorAdapter] def getClusterDescriptorAdapter: T = clusterDescriptor + def setSubmitInterceptor(interceptor: FlinkJobSubmitInterceptor): Unit = { + this.interceptor = interceptor + } + override def getId: String = "FlinkOnceApp_" + id protected def closeDaemon(): Unit = { diff --git a/linkis-engineconn-plugins/flink/src/main/scala/org/apache/linkis/engineconnplugin/flink/executor/interceptor/FlinkJobResCleanInterceptor.scala b/linkis-engineconn-plugins/flink/src/main/scala/org/apache/linkis/engineconnplugin/flink/executor/interceptor/FlinkJobResCleanInterceptor.scala new file mode 100644 index 0000000000..9980e8edaf --- /dev/null +++ b/linkis-engineconn-plugins/flink/src/main/scala/org/apache/linkis/engineconnplugin/flink/executor/interceptor/FlinkJobResCleanInterceptor.scala @@ -0,0 +1,89 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.engineconnplugin.flink.executor.interceptor + +import org.apache.linkis.engineconn.once.executor.OnceExecutorExecutionContext +import org.apache.linkis.engineconnplugin.flink.config.FlinkEnvConfiguration.{FLINK_APPLICATION_MAIN_CLASS_JAR, FLINK_SHIP_DIRECTORIES} +import org.apache.linkis.engineconnplugin.flink.resource.FlinkJobResourceCleaner +import org.apache.commons.lang3.StringUtils +import org.apache.linkis.common.utils.Logging + +import java.util + +/** + * Flink Job resource clean interceptor + */ +class FlinkJobResCleanInterceptor(cleaner: FlinkJobResourceCleaner) + extends FlinkJobSubmitInterceptor with Logging{ + + /** + * Before submit the job + * + * @param onceExecutorExecutionContext + * execution context + */ + override def beforeSubmit(onceExecutorExecutionContext: OnceExecutorExecutionContext): Unit = { + // Do nothing + + } + + /** + * After success to submit the job + * + * @param onceExecutorExecutionContext + * execution context + */ + override def afterSubmitSuccess( + onceExecutorExecutionContext: OnceExecutorExecutionContext + ): Unit = { + logger.info("Clean the flink job resource after success to submit") + cleanResources(onceExecutorExecutionContext.getEngineCreationContext.getOptions) + } + + /** + * After fail to submit the job + * + * @param onceExecutorExecutionContext + * execution context + * @param throwable + * throwable + */ + override def afterSubmitFail( + onceExecutorExecutionContext: OnceExecutorExecutionContext, + throwable: Throwable + ): Unit = { + logger.info("Clean the flink job resource after fail to submit") + cleanResources(onceExecutorExecutionContext.getEngineCreationContext.getOptions) + } + + private def cleanResources(options: util.Map[String, String]): Unit = { + val mainClassJar = FLINK_APPLICATION_MAIN_CLASS_JAR.getValue(options) + logger.trace(s"mainClassJar to clean: ${mainClassJar}") + if (StringUtils.isNotBlank(mainClassJar) && cleaner.accept(mainClassJar)) { + cleaner.cleanup(Array(mainClassJar)) + } + val shipDirsArray = (FLINK_SHIP_DIRECTORIES.getValue(options) + "," + + FLINK_SHIP_DIRECTORIES.getValue).split(",") + logger.trace(s"Ship directories to clean: ${shipDirsArray.length}") + shipDirsArray match { + case resArray: Array[String] => cleaner.cleanup(resArray.filter(cleaner.accept)) + case _ => + } + } + +} diff --git a/linkis-engineconn-plugins/flink/src/main/scala/org/apache/linkis/engineconnplugin/flink/executor/interceptor/FlinkJobSubmitInterceptor.scala b/linkis-engineconn-plugins/flink/src/main/scala/org/apache/linkis/engineconnplugin/flink/executor/interceptor/FlinkJobSubmitInterceptor.scala new file mode 100644 index 0000000000..1c11cc9d0e --- /dev/null +++ b/linkis-engineconn-plugins/flink/src/main/scala/org/apache/linkis/engineconnplugin/flink/executor/interceptor/FlinkJobSubmitInterceptor.scala @@ -0,0 +1,50 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.engineconnplugin.flink.executor.interceptor + +import org.apache.linkis.engineconn.once.executor.OnceExecutorExecutionContext + +trait FlinkJobSubmitInterceptor { + + /** + * Before submit the job + * @param onceExecutorExecutionContext + * execution context + */ + def beforeSubmit(onceExecutorExecutionContext: OnceExecutorExecutionContext): Unit + + /** + * After success to submit the job + * @param onceExecutorExecutionContext + * execution context + */ + def afterSubmitSuccess(onceExecutorExecutionContext: OnceExecutorExecutionContext): Unit + + /** + * After fail to submit the job + * @param onceExecutorExecutionContext + * execution context + * @param throwable + * throwable + */ + def afterSubmitFail( + onceExecutorExecutionContext: OnceExecutorExecutionContext, + throwable: Throwable + ): Unit + +} diff --git a/linkis-engineconn-plugins/flink/src/main/scala/org/apache/linkis/engineconnplugin/flink/factory/FlinkApplicationExecutorFactory.scala b/linkis-engineconn-plugins/flink/src/main/scala/org/apache/linkis/engineconnplugin/flink/factory/FlinkApplicationExecutorFactory.scala index 1b8c2ad5b5..a5476edf88 100644 --- a/linkis-engineconn-plugins/flink/src/main/scala/org/apache/linkis/engineconnplugin/flink/factory/FlinkApplicationExecutorFactory.scala +++ b/linkis-engineconn-plugins/flink/src/main/scala/org/apache/linkis/engineconnplugin/flink/factory/FlinkApplicationExecutorFactory.scala @@ -23,12 +23,20 @@ import org.apache.linkis.engineconn.once.executor.OnceExecutor import org.apache.linkis.engineconn.once.executor.creation.OnceExecutorFactory import org.apache.linkis.engineconnplugin.flink.context.FlinkEngineConnContext import org.apache.linkis.engineconnplugin.flink.executor.FlinkJarOnceExecutor +import org.apache.linkis.engineconnplugin.flink.executor.interceptor.FlinkJobResCleanInterceptor +import org.apache.linkis.engineconnplugin.flink.resource.FlinkJobLocalResourceCleaner import org.apache.linkis.manager.label.entity.Label import org.apache.linkis.manager.label.entity.engine.RunType import org.apache.linkis.manager.label.entity.engine.RunType.RunType class FlinkApplicationExecutorFactory extends OnceExecutorFactory { + /** + * Interceptor + * @return + */ + private def submitInterceptor = new FlinkJobResCleanInterceptor(new FlinkJobLocalResourceCleaner) + override protected def newExecutor( id: Int, engineCreationContext: EngineCreationContext, @@ -36,7 +44,9 @@ class FlinkApplicationExecutorFactory extends OnceExecutorFactory { labels: Array[Label[_]] ): OnceExecutor = engineConn.getEngineConnSession match { case context: FlinkEngineConnContext => - new FlinkJarOnceExecutor(id, context) + val executor = new FlinkJarOnceExecutor(id, context) + executor.setSubmitInterceptor(submitInterceptor) + executor } override protected def getRunType: RunType = RunType.JAR diff --git a/linkis-engineconn-plugins/flink/src/main/scala/org/apache/linkis/engineconnplugin/flink/resource/FlinkJobLocalResourceCleaner.scala b/linkis-engineconn-plugins/flink/src/main/scala/org/apache/linkis/engineconnplugin/flink/resource/FlinkJobLocalResourceCleaner.scala new file mode 100644 index 0000000000..f18eda9d1e --- /dev/null +++ b/linkis-engineconn-plugins/flink/src/main/scala/org/apache/linkis/engineconnplugin/flink/resource/FlinkJobLocalResourceCleaner.scala @@ -0,0 +1,70 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.engineconnplugin.flink.resource + +import org.apache.linkis.common.utils.Logging + +import java.io.File + +/** + * Cleaner for local resource + */ +class FlinkJobLocalResourceCleaner extends FlinkJobResourceCleaner with Logging{ + + /** + * Clean up path array + * + * @param resArray + * resource array + */ + override def cleanup(resArray: Array[String]): Unit = { + Option(resArray).foreach(paths => + paths.foreach(path => { + Option(getClass.getClassLoader.getResource(path)) match { + case Some(url) => + val localFile = new File(url.getPath) + if (localFile.exists()) { + logger.info(s"Clean the resource: [${localFile.getPath}]") + localFile.delete() + } + case _ => + } + + }) + ) + } + + /** + * If accept the path + * + * @param resource + * path + * @return + */ + override def accept(resource: String): Boolean = { + Option(getClass.getClassLoader.getResource(resource)) match { + case Some(url) => + val file = new File(url.getPath) + // Avoid the linked file/directory + file.getCanonicalFile.equals(file.getAbsoluteFile) + case _ => false + } + } + +} + diff --git a/linkis-engineconn-plugins/flink/src/main/scala/org/apache/linkis/engineconnplugin/flink/resource/FlinkJobResourceCleaner.scala b/linkis-engineconn-plugins/flink/src/main/scala/org/apache/linkis/engineconnplugin/flink/resource/FlinkJobResourceCleaner.scala new file mode 100644 index 0000000000..76d4fe8552 --- /dev/null +++ b/linkis-engineconn-plugins/flink/src/main/scala/org/apache/linkis/engineconnplugin/flink/resource/FlinkJobResourceCleaner.scala @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.engineconnplugin.flink.resource + +trait FlinkJobResourceCleaner { + + /** + * Clean up resource array + * + * @param resArray + * resource array + */ + def cleanup(resArray: Array[String]): Unit + + /** + * If accept the resource + * + * @param resource + * resource + * @return + */ + def accept(resource: String): Boolean +} From 4f63e069b86a9497928be919844d550922b45dc0 Mon Sep 17 00:00:00 2001 From: davidhua Date: Tue, 15 Nov 2022 20:24:53 +0800 Subject: [PATCH 009/261] Exclude the FLINK_SHIP_DIRECTORIES from server in cleaner of Flink Engine. --- .../executor/interceptor/FlinkJobResCleanInterceptor.scala | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/linkis-engineconn-plugins/flink/src/main/scala/org/apache/linkis/engineconnplugin/flink/executor/interceptor/FlinkJobResCleanInterceptor.scala b/linkis-engineconn-plugins/flink/src/main/scala/org/apache/linkis/engineconnplugin/flink/executor/interceptor/FlinkJobResCleanInterceptor.scala index 9980e8edaf..c5caf5bdfb 100644 --- a/linkis-engineconn-plugins/flink/src/main/scala/org/apache/linkis/engineconnplugin/flink/executor/interceptor/FlinkJobResCleanInterceptor.scala +++ b/linkis-engineconn-plugins/flink/src/main/scala/org/apache/linkis/engineconnplugin/flink/executor/interceptor/FlinkJobResCleanInterceptor.scala @@ -77,8 +77,7 @@ class FlinkJobResCleanInterceptor(cleaner: FlinkJobResourceCleaner) if (StringUtils.isNotBlank(mainClassJar) && cleaner.accept(mainClassJar)) { cleaner.cleanup(Array(mainClassJar)) } - val shipDirsArray = (FLINK_SHIP_DIRECTORIES.getValue(options) + "," + - FLINK_SHIP_DIRECTORIES.getValue).split(",") + val shipDirsArray = FLINK_SHIP_DIRECTORIES.getValue(options).split(",") logger.trace(s"Ship directories to clean: ${shipDirsArray.length}") shipDirsArray match { case resArray: Array[String] => cleaner.cleanup(resArray.filter(cleaner.accept)) From f314e48198ba0e348f51f0baff8899dd84dac207 Mon Sep 17 00:00:00 2001 From: peacewong Date: Thu, 16 Mar 2023 16:31:02 +0800 Subject: [PATCH 010/261] code format --- .../engineplugin/common/util/NodeResourceUtils.scala | 2 +- .../interceptor/FlinkJobResCleanInterceptor.scala | 11 ++++++++--- .../flink/resource/FlinkJobLocalResourceCleaner.scala | 3 +-- 3 files changed, 10 insertions(+), 6 deletions(-) diff --git a/linkis-computation-governance/linkis-engineconn/linkis-engineconn-plugin-core/src/main/scala/org/apache/linkis/manager/engineplugin/common/util/NodeResourceUtils.scala b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-plugin-core/src/main/scala/org/apache/linkis/manager/engineplugin/common/util/NodeResourceUtils.scala index 37ba218426..29f4c954f4 100644 --- a/linkis-computation-governance/linkis-engineconn/linkis-engineconn-plugin-core/src/main/scala/org/apache/linkis/manager/engineplugin/common/util/NodeResourceUtils.scala +++ b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-plugin-core/src/main/scala/org/apache/linkis/manager/engineplugin/common/util/NodeResourceUtils.scala @@ -23,8 +23,8 @@ import org.apache.linkis.manager.engineplugin.common.conf.EngineConnPluginConf import org.apache.commons.lang3.StringUtils import java.util -import java.util.regex.Pattern import java.util.{Locale, Objects} +import java.util.regex.Pattern import org.slf4j.LoggerFactory diff --git a/linkis-engineconn-plugins/flink/src/main/scala/org/apache/linkis/engineconnplugin/flink/executor/interceptor/FlinkJobResCleanInterceptor.scala b/linkis-engineconn-plugins/flink/src/main/scala/org/apache/linkis/engineconnplugin/flink/executor/interceptor/FlinkJobResCleanInterceptor.scala index c5caf5bdfb..b33f005ca9 100644 --- a/linkis-engineconn-plugins/flink/src/main/scala/org/apache/linkis/engineconnplugin/flink/executor/interceptor/FlinkJobResCleanInterceptor.scala +++ b/linkis-engineconn-plugins/flink/src/main/scala/org/apache/linkis/engineconnplugin/flink/executor/interceptor/FlinkJobResCleanInterceptor.scala @@ -17,11 +17,15 @@ package org.apache.linkis.engineconnplugin.flink.executor.interceptor +import org.apache.linkis.common.utils.Logging import org.apache.linkis.engineconn.once.executor.OnceExecutorExecutionContext -import org.apache.linkis.engineconnplugin.flink.config.FlinkEnvConfiguration.{FLINK_APPLICATION_MAIN_CLASS_JAR, FLINK_SHIP_DIRECTORIES} +import org.apache.linkis.engineconnplugin.flink.config.FlinkEnvConfiguration.{ + FLINK_APPLICATION_MAIN_CLASS_JAR, + FLINK_SHIP_DIRECTORIES +} import org.apache.linkis.engineconnplugin.flink.resource.FlinkJobResourceCleaner + import org.apache.commons.lang3.StringUtils -import org.apache.linkis.common.utils.Logging import java.util @@ -29,7 +33,8 @@ import java.util * Flink Job resource clean interceptor */ class FlinkJobResCleanInterceptor(cleaner: FlinkJobResourceCleaner) - extends FlinkJobSubmitInterceptor with Logging{ + extends FlinkJobSubmitInterceptor + with Logging { /** * Before submit the job diff --git a/linkis-engineconn-plugins/flink/src/main/scala/org/apache/linkis/engineconnplugin/flink/resource/FlinkJobLocalResourceCleaner.scala b/linkis-engineconn-plugins/flink/src/main/scala/org/apache/linkis/engineconnplugin/flink/resource/FlinkJobLocalResourceCleaner.scala index f18eda9d1e..b849d6967d 100644 --- a/linkis-engineconn-plugins/flink/src/main/scala/org/apache/linkis/engineconnplugin/flink/resource/FlinkJobLocalResourceCleaner.scala +++ b/linkis-engineconn-plugins/flink/src/main/scala/org/apache/linkis/engineconnplugin/flink/resource/FlinkJobLocalResourceCleaner.scala @@ -24,7 +24,7 @@ import java.io.File /** * Cleaner for local resource */ -class FlinkJobLocalResourceCleaner extends FlinkJobResourceCleaner with Logging{ +class FlinkJobLocalResourceCleaner extends FlinkJobResourceCleaner with Logging { /** * Clean up path array @@ -67,4 +67,3 @@ class FlinkJobLocalResourceCleaner extends FlinkJobResourceCleaner with Logging{ } } - From c448d50ab4509d7a3ba99f9d527ece8744732d60 Mon Sep 17 00:00:00 2001 From: alexkun Date: Tue, 6 Dec 2022 21:02:31 +0800 Subject: [PATCH 011/261] engineconn-plugin-core - add config for MaxMetaspaceSize --- .../engineplugin/common/conf/EnvConfiguration.scala | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/linkis-computation-governance/linkis-engineconn/linkis-engineconn-plugin-core/src/main/scala/org/apache/linkis/manager/engineplugin/common/conf/EnvConfiguration.scala b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-plugin-core/src/main/scala/org/apache/linkis/manager/engineplugin/common/conf/EnvConfiguration.scala index 290c6211e1..63f13dc5f3 100644 --- a/linkis-computation-governance/linkis-engineconn/linkis-engineconn-plugin-core/src/main/scala/org/apache/linkis/manager/engineplugin/common/conf/EnvConfiguration.scala +++ b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-plugin-core/src/main/scala/org/apache/linkis/manager/engineplugin/common/conf/EnvConfiguration.scala @@ -41,13 +41,15 @@ object EnvConfiguration { val ENGINE_CONN_CLASSPATH_FILES = CommonVars("wds.linkis.engineConn.files", "", "engineConn额外的配置文件") - val metaspaceSize = if (SystemUtils.isJavaVersionAtLeast(JavaVersion.JAVA_1_8)) { - "-XX:MaxMetaspaceSize=256m -XX:MetaspaceSize=128m" + val MAX_METASPACE_SIZE = CommonVars("linkis.engineconn.metaspace.size.max", "256m") + + lazy val metaspaceSize = if (SystemUtils.isJavaVersionAtLeast(JavaVersion.JAVA_1_8)) { + s"-XX:MaxMetaspaceSize=${MAX_METASPACE_SIZE.getHotValue()} -XX:MetaspaceSize=128m" } else { - "-XX:MaxPermSize=256m -XX:PermSize=128m" + s"-XX:MaxPermSize=${MAX_METASPACE_SIZE.getHotValue()} -XX:PermSize=128m" } - val ENGINE_CONN_DEFAULT_JAVA_OPTS = CommonVars[String]( + lazy val ENGINE_CONN_DEFAULT_JAVA_OPTS = CommonVars[String]( "wds.linkis.engineConn.javaOpts.default", s"-XX:+UseG1GC ${metaspaceSize} " + s"-Xloggc:%s -XX:+PrintGCDetails -XX:+PrintGCTimeStamps -XX:+PrintGCDateStamps -Dwds.linkis.server.conf=linkis-engineconn.properties -Dwds.linkis.gateway.url=${Configuration.getGateWayURL()}" From b783bfe2099d3f999c35b46d241a1be2f53071da Mon Sep 17 00:00:00 2001 From: alexkun Date: Thu, 15 Dec 2022 21:58:53 +0800 Subject: [PATCH 012/261] Merge branch 'dev-1.3.2-part2-fix-metaspace-oom-not-exit' into dev-1.1.8-webank From 96632b50b2f8fcca4a33ddee5178ff0a0a2c3e92 Mon Sep 17 00:00:00 2001 From: peacewong Date: Mon, 20 Mar 2023 11:22:59 +0800 Subject: [PATCH 013/261] remove jdk8 judge --- .../engineplugin/common/conf/EnvConfiguration.scala | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/linkis-computation-governance/linkis-engineconn/linkis-engineconn-plugin-core/src/main/scala/org/apache/linkis/manager/engineplugin/common/conf/EnvConfiguration.scala b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-plugin-core/src/main/scala/org/apache/linkis/manager/engineplugin/common/conf/EnvConfiguration.scala index 63f13dc5f3..f3235ffa34 100644 --- a/linkis-computation-governance/linkis-engineconn/linkis-engineconn-plugin-core/src/main/scala/org/apache/linkis/manager/engineplugin/common/conf/EnvConfiguration.scala +++ b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-plugin-core/src/main/scala/org/apache/linkis/manager/engineplugin/common/conf/EnvConfiguration.scala @@ -43,11 +43,8 @@ object EnvConfiguration { val MAX_METASPACE_SIZE = CommonVars("linkis.engineconn.metaspace.size.max", "256m") - lazy val metaspaceSize = if (SystemUtils.isJavaVersionAtLeast(JavaVersion.JAVA_1_8)) { - s"-XX:MaxMetaspaceSize=${MAX_METASPACE_SIZE.getHotValue()} -XX:MetaspaceSize=128m" - } else { - s"-XX:MaxPermSize=${MAX_METASPACE_SIZE.getHotValue()} -XX:PermSize=128m" - } + lazy val metaspaceSize = + s"-XX:MaxMetaspaceSize=${MAX_METASPACE_SIZE.getValue} -XX:MetaspaceSize=128m" lazy val ENGINE_CONN_DEFAULT_JAVA_OPTS = CommonVars[String]( "wds.linkis.engineConn.javaOpts.default", From bfcc534d90df4cfc48c93ba083a639f944d8ea5f Mon Sep 17 00:00:00 2001 From: huangKai-2323 <62878639+huangKai-2323@users.noreply.github.com> Date: Mon, 20 Mar 2023 20:58:57 +0800 Subject: [PATCH 014/261] optimize engine log (#98) --- .../server/service/impl/DefaultEngineConnKillService.java | 2 +- .../launch/process/JavaProcessEngineConnLaunchBuilder.scala | 2 +- .../org/apache/linkis/manager/am/restful/EMRestfulApi.java | 5 ++++- linkis-engineconn-plugins/hive/src/main/resources/log4j2.xml | 2 +- .../shell/src/main/resources/conf/log4j2.xml | 2 +- .../spark/src/main/resources/log4j2.xml | 2 +- .../sqoop/src/main/resources/log4j2.xml | 2 +- 7 files changed, 10 insertions(+), 7 deletions(-) diff --git a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/service/impl/DefaultEngineConnKillService.java b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/service/impl/DefaultEngineConnKillService.java index 4f593736d9..cb8c7b1161 100644 --- a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/service/impl/DefaultEngineConnKillService.java +++ b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/service/impl/DefaultEngineConnKillService.java @@ -98,7 +98,7 @@ public void killYarnAppIdOfOneEc(EngineConn engineConn) { String engineConnInstance = engineConn.getServiceInstance().toString(); logger.info("try to kill yarn app ids in the engine of ({}).", engineConnInstance); String engineLogDir = engineConn.getEngineConnManagerEnv().engineConnLogDirs(); - final String errEngineLogPath = engineLogDir.concat(File.separator).concat("yarnApp.log"); + final String errEngineLogPath = engineLogDir.concat(File.separator).concat("yarnApp"); logger.info("try to parse the yarn app id from the engine err log file path: {}", errEngineLogPath); File file = new File(errEngineLogPath); if (file.exists()) diff --git a/linkis-computation-governance/linkis-engineconn/linkis-engineconn-plugin-core/src/main/scala/org/apache/linkis/manager/engineplugin/common/launch/process/JavaProcessEngineConnLaunchBuilder.scala b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-plugin-core/src/main/scala/org/apache/linkis/manager/engineplugin/common/launch/process/JavaProcessEngineConnLaunchBuilder.scala index 5271ec37e9..082b02a020 100644 --- a/linkis-computation-governance/linkis-engineconn/linkis-engineconn-plugin-core/src/main/scala/org/apache/linkis/manager/engineplugin/common/launch/process/JavaProcessEngineConnLaunchBuilder.scala +++ b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-plugin-core/src/main/scala/org/apache/linkis/manager/engineplugin/common/launch/process/JavaProcessEngineConnLaunchBuilder.scala @@ -53,7 +53,7 @@ abstract class JavaProcessEngineConnLaunchBuilder this.engineConnResourceGenerator = engineConnResourceGenerator protected def getGcLogDir(engineConnBuildRequest: EngineConnBuildRequest): String = - variable(LOG_DIRS) + "/gc.log" + variable(LOG_DIRS) + "/gc" protected def getLogDir(engineConnBuildRequest: EngineConnBuildRequest): String = s" -Dlogging.file=${EnvConfiguration.LOG4J2_XML_FILE.getValue} " + diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/restful/EMRestfulApi.java b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/restful/EMRestfulApi.java index 4d8bbad27d..4477d60ef7 100644 --- a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/restful/EMRestfulApi.java +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/restful/EMRestfulApi.java @@ -392,7 +392,10 @@ public Message openEngineLog(HttpServletRequest req, @RequestBody JsonNode jsonN jsonNode.get("parameters").toString(), new TypeReference>() {}); String logType = (String) parameters.get("logType"); - if (!logType.equals("stdout") && !logType.equals("stderr")) { + if (!logType.equals("stdout") + && !logType.equals("stderr") + && !logType.equals("gc") + && !logType.equals("yarnApp")) { throw new AMErrorException( AMErrorCode.PARAM_ERROR.getErrorCode(), AMErrorCode.PARAM_ERROR.getErrorDesc()); } diff --git a/linkis-engineconn-plugins/hive/src/main/resources/log4j2.xml b/linkis-engineconn-plugins/hive/src/main/resources/log4j2.xml index e6f8b150f3..79934d9832 100644 --- a/linkis-engineconn-plugins/hive/src/main/resources/log4j2.xml +++ b/linkis-engineconn-plugins/hive/src/main/resources/log4j2.xml @@ -27,7 +27,7 @@ - + diff --git a/linkis-engineconn-plugins/shell/src/main/resources/conf/log4j2.xml b/linkis-engineconn-plugins/shell/src/main/resources/conf/log4j2.xml index 68173f23ac..50b88bf799 100644 --- a/linkis-engineconn-plugins/shell/src/main/resources/conf/log4j2.xml +++ b/linkis-engineconn-plugins/shell/src/main/resources/conf/log4j2.xml @@ -27,7 +27,7 @@ - + diff --git a/linkis-engineconn-plugins/spark/src/main/resources/log4j2.xml b/linkis-engineconn-plugins/spark/src/main/resources/log4j2.xml index b02d5889cd..f41859dafb 100644 --- a/linkis-engineconn-plugins/spark/src/main/resources/log4j2.xml +++ b/linkis-engineconn-plugins/spark/src/main/resources/log4j2.xml @@ -27,7 +27,7 @@ - + diff --git a/linkis-engineconn-plugins/sqoop/src/main/resources/log4j2.xml b/linkis-engineconn-plugins/sqoop/src/main/resources/log4j2.xml index 9fd15a4b48..d93effc17e 100644 --- a/linkis-engineconn-plugins/sqoop/src/main/resources/log4j2.xml +++ b/linkis-engineconn-plugins/sqoop/src/main/resources/log4j2.xml @@ -34,7 +34,7 @@ - + From ebba86b317de48e01e775880cf0f605d2e2b122a Mon Sep 17 00:00:00 2001 From: huangKai-2323 <62878639+huangKai-2323@users.noreply.github.com> Date: Mon, 20 Mar 2023 21:00:00 +0800 Subject: [PATCH 015/261] update task cancel msg (#97) --- .../apache/linkis/entrance/server/DefaultEntranceServer.java | 3 ++- .../jobhistory/errorcode/JobhistoryErrorCodeSummary.java | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/server/DefaultEntranceServer.java b/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/server/DefaultEntranceServer.java index 4301334965..ddf8e45767 100644 --- a/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/server/DefaultEntranceServer.java +++ b/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/server/DefaultEntranceServer.java @@ -98,7 +98,8 @@ private void shutdownEntrance(ContextClosedEvent event) { if (null != allUndoneJobs) { for (EntranceJob job : allUndoneJobs) { job.onFailure( - "Entrance exits the automatic cleanup task and can be rerun(服务退出自动清理任务，可以重跑)", null); + "Your job will be marked as canceled because the Entrance service restarted(因为Entrance服务重启，您的任务将被标记为取消)", + null); } } } diff --git a/linkis-public-enhancements/linkis-jobhistory/src/main/java/org/apache/linkis/jobhistory/errorcode/JobhistoryErrorCodeSummary.java b/linkis-public-enhancements/linkis-jobhistory/src/main/java/org/apache/linkis/jobhistory/errorcode/JobhistoryErrorCodeSummary.java index 5c62944d5e..3cd7137261 100644 --- a/linkis-public-enhancements/linkis-jobhistory/src/main/java/org/apache/linkis/jobhistory/errorcode/JobhistoryErrorCodeSummary.java +++ b/linkis-public-enhancements/linkis-jobhistory/src/main/java/org/apache/linkis/jobhistory/errorcode/JobhistoryErrorCodeSummary.java @@ -22,7 +22,7 @@ public enum JobhistoryErrorCodeSummary implements LinkisErrorCode { UNFINISHED_TASKS( 20020, - "entrance service restart, automatically cancel tasks that have not been completed for a long time (24h)"); + "Your job will be marked as canceled because the Entrance service restarted(因为Entrance服务重启，您的任务将被标记为取消)"); /** (errorCode)错误码 */ private final int errorCode; From 239fbb73abcc67c9e113b82c0cb420da39845ca6 Mon Sep 17 00:00:00 2001 From: huangKai-2323 <62878639+huangKai-2323@users.noreply.github.com> Date: Mon, 20 Mar 2023 21:00:31 +0800 Subject: [PATCH 016/261] update log (#94) --- .../org/apache/linkis/udf/service/impl/UDFTreeServiceImpl.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/linkis-public-enhancements/linkis-udf/linkis-udf-service/src/main/java/org/apache/linkis/udf/service/impl/UDFTreeServiceImpl.java b/linkis-public-enhancements/linkis-udf/linkis-udf-service/src/main/java/org/apache/linkis/udf/service/impl/UDFTreeServiceImpl.java index 375d9c6cba..0f3b60ff5e 100644 --- a/linkis-public-enhancements/linkis-udf/linkis-udf-service/src/main/java/org/apache/linkis/udf/service/impl/UDFTreeServiceImpl.java +++ b/linkis-public-enhancements/linkis-udf/linkis-udf-service/src/main/java/org/apache/linkis/udf/service/impl/UDFTreeServiceImpl.java @@ -114,7 +114,7 @@ public UDFTree getFirstFloor(String userName, String category) throws UDFExcepti } if (first.size() > 1) { throw new UDFException( - "user(用户)" + userName + "There are two root directory directories(存在两个根目录目录)"); + "user(用户):" + userName + ",There are two root directory directories(存在两个根目录目录)"); } return first.get(0); } From 2d89558813f11a9da135e7a06b3ccdbe8b3a7140 Mon Sep 17 00:00:00 2001 From: huangKai-2323 <62878639+huangKai-2323@users.noreply.github.com> Date: Mon, 20 Mar 2023 21:10:19 +0800 Subject: [PATCH 017/261] fix shell upload blocking (#95) --- .../main/scala/org/apache/linkis/common/utils/Utils.scala | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/linkis-commons/linkis-common/src/main/scala/org/apache/linkis/common/utils/Utils.scala b/linkis-commons/linkis-common/src/main/scala/org/apache/linkis/common/utils/Utils.scala index 80e3ff7e5e..d469f60fba 100644 --- a/linkis-commons/linkis-common/src/main/scala/org/apache/linkis/common/utils/Utils.scala +++ b/linkis-commons/linkis-common/src/main/scala/org/apache/linkis/common/utils/Utils.scala @@ -304,6 +304,13 @@ object Utils extends Logging { pb.redirectInput(ProcessBuilder.Redirect.PIPE) val process = pb.start val log = new BufferedReader(new InputStreamReader(process.getInputStream)) + defaultScheduler.submit(new Runnable { + override def run(): Unit = { + while (process.isAlive) { + logger.info(log.readLine()) + } + } + }) val exitCode = if (maxWaitTime > 0) { val completed = process.waitFor(maxWaitTime, TimeUnit.MILLISECONDS) if (!completed) { From a18fccedaf913d54b45fa5e14c3a396ea9c874c9 Mon Sep 17 00:00:00 2001 From: huangKai-2323 <62878639+huangKai-2323@users.noreply.github.com> Date: Mon, 20 Mar 2023 21:38:50 +0800 Subject: [PATCH 018/261] =?UTF-8?q?=E3=80=901.1.10=E3=80=91Fix=20the=20gar?= =?UTF-8?q?bled=20code=20problem=20of=20spark=20function=20(#93)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix stddev_samp bug * Code optimization --- .../org/apache/linkis/storage/domain/DataType.scala | 9 ++++++++- .../linkis/engineplugin/spark/executor/SQLSession.scala | 7 ++++++- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/domain/DataType.scala b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/domain/DataType.scala index 55c82abd38..59d8860c1c 100644 --- a/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/domain/DataType.scala +++ b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/domain/DataType.scala @@ -79,7 +79,14 @@ object DataType extends Logging { case IntType => if (isNumberNull(value)) null else value.toInt case LongType | BigIntType => if (isNumberNull(value)) null else value.toLong case FloatType => if (isNumberNull(value)) null else value.toFloat - case DoubleType => if (isNumberNull(value)) null else value.toDouble + case DoubleType => + if (isNumberNull(value)) { + null + } else if (value.asInstanceOf[Double].isNaN) { + "NaN" + } else { + value.toDouble + } case DecimalType => if (isNumberNull(value)) null else new JavaBigDecimal(value) case DateType => if (isNumberNull(value)) null else Date.valueOf(value) case TimestampType => diff --git a/linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/executor/SQLSession.scala b/linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/executor/SQLSession.scala index 3291304868..9671ab6922 100644 --- a/linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/executor/SQLSession.scala +++ b/linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/executor/SQLSession.scala @@ -214,7 +214,12 @@ object SQLSession extends Logging { .mkString("{", ",", "}") case (str: String, StringType) => str.replaceAll("\n|\t", " ") - case (double: Double, DoubleType) => nf.format(double) + case (double: Double, DoubleType) => + if (double.isNaN) { + "NaN" + } else { + nf.format(double) + } case (decimal: java.math.BigDecimal, DecimalType()) => formatDecimal(decimal) case (other: Any, tpe) => other.toString case _ => null From cc5f5233e16c27c0bd623577deee4d9874713f92 Mon Sep 17 00:00:00 2001 From: casionone Date: Mon, 20 Mar 2023 23:11:10 +0800 Subject: [PATCH 019/261] set version 1.1.10-wedatasphere --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 5b9cb09c94..0eaa57f1f5 100644 --- a/pom.xml +++ b/pom.xml @@ -103,7 +103,7 @@ - 1.3.2-SNAPSHOT + 1.1.10-wedatasphere 2.9.2 2.4.3 2.7.2 From 90efd0896d23184a2362d9940dfff52fcc9175a5 Mon Sep 17 00:00:00 2001 From: huangKai-2323 <62878639+huangKai-2323@users.noreply.github.com> Date: Wed, 22 Mar 2023 19:26:53 +0800 Subject: [PATCH 020/261] =?UTF-8?q?=E3=80=901.1.10=E3=80=91Repair=20of=20t?= =?UTF-8?q?he=20parameter=20validation=20problem=20configured=20by=20the?= =?UTF-8?q?=20management=20console=20(#96)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix config cache bug * code optimize * Code optimization * optimize code --------- Co-authored-by: Casion --- docs/info-1.3.2.md | 1 + .../restful/api/ConfigurationRestfulApi.java | 32 +++++++++++++++++-- .../configuration/conf/Configuration.scala | 3 ++ 3 files changed, 33 insertions(+), 3 deletions(-) diff --git a/docs/info-1.3.2.md b/docs/info-1.3.2.md index 55e5aeecbc..0c690517fb 100644 --- a/docs/info-1.3.2.md +++ b/docs/info-1.3.2.md @@ -4,4 +4,5 @@ |------------------| ----- |----------------------------------------------------------------------|------| ------------------------------------------------------- | | linkis-jobhistory | 新增 | wds.linkis.jobhistory.admin | hadoop |可以查看所有历史任务的用户 注意：wds.linkis.governance.station.admin 为管理用户（也具有可以查看所有历史任务的权限）| | linkis | 新增 | wds.linkis.governance.station.admin.token | /具有管理员权限的特殊token| +| linkis | 新增 | linkis.configuration.remove.application.cache | IDE |清除该应用的配置缓存| | cg-entrance | 新增 | linkis.entrance.auto.clean.dirty.data.enable | true |entrance重启调用ps-jobhistory接口是否开启，ture为开启，取值范围：true或false| diff --git a/linkis-public-enhancements/linkis-configuration/src/main/java/org/apache/linkis/configuration/restful/api/ConfigurationRestfulApi.java b/linkis-public-enhancements/linkis-configuration/src/main/java/org/apache/linkis/configuration/restful/api/ConfigurationRestfulApi.java index dd4bbc258b..fe1bd36b57 100644 --- a/linkis-public-enhancements/linkis-configuration/src/main/java/org/apache/linkis/configuration/restful/api/ConfigurationRestfulApi.java +++ b/linkis-public-enhancements/linkis-configuration/src/main/java/org/apache/linkis/configuration/restful/api/ConfigurationRestfulApi.java @@ -275,9 +275,35 @@ public Message saveFullTree(HttpServletRequest req, @RequestBody JsonNode json) version = tmpString[1]; } configurationService.updateUserValue(createList, updateList); - configurationService.clearAMCacheConf(username, creator, engine, version); - Message message = Message.ok(); - return message; + // TODO: Add a refresh cache interface later + if (StringUtils.isNotBlank(creator) && creator.equals("*")) { + List allCategory = categoryService.getAllCategory(null); + List categoryLabelVos = + allCategory.stream() + .filter(s -> s.getCategoryName().equals(Configuration.REMOVE_APPLICATION_CACHE())) + .map(CategoryLabelVo::getChildCategory) + .findFirst() + .get(); + categoryLabelVos.stream() + .map(CategoryLabelVo::getCategoryName) + .filter(StringUtils::isNotBlank) + .forEach( + info -> { + String[] tmpString = info.split("-"); + if (tmpString.length == 2) { + String engineName = tmpString[0]; + String engineVersion = tmpString[1]; + configurationService.clearAMCacheConf( + username, + Configuration.REMOVE_APPLICATION_CACHE(), + engineName, + engineVersion); + } + }); + } else { + configurationService.clearAMCacheConf(username, creator, engine, version); + } + return Message.ok(); } @ApiOperation( diff --git a/linkis-public-enhancements/linkis-configuration/src/main/scala/org/apache/linkis/configuration/conf/Configuration.scala b/linkis-public-enhancements/linkis-configuration/src/main/scala/org/apache/linkis/configuration/conf/Configuration.scala index 1721241ab1..963837adef 100644 --- a/linkis-public-enhancements/linkis-configuration/src/main/scala/org/apache/linkis/configuration/conf/Configuration.scala +++ b/linkis-public-enhancements/linkis-configuration/src/main/scala/org/apache/linkis/configuration/conf/Configuration.scala @@ -43,4 +43,7 @@ object Configuration { val USE_CREATOR_DEFAULE_VALUE = CommonVars.apply("wds.linkis.configuration.use.creator.default.value", true).getValue + val REMOVE_APPLICATION_CACHE = + CommonVars.apply("linkis.configuration.remove.application.cache", "IDE").getValue + } From c0df23ab422a716aec2eaa09f4ed2a1f14368a18 Mon Sep 17 00:00:00 2001 From: Casion Date: Wed, 22 Mar 2023 19:32:28 +0800 Subject: [PATCH 021/261] fix yarn resource add + not correct (#102) --- .../manager/common/entity/resource/Resource.scala | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/scala/org/apache/linkis/manager/common/entity/resource/Resource.scala b/linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/scala/org/apache/linkis/manager/common/entity/resource/Resource.scala index 47b6515849..0cfb4ae055 100644 --- a/linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/scala/org/apache/linkis/manager/common/entity/resource/Resource.scala +++ b/linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/scala/org/apache/linkis/manager/common/entity/resource/Resource.scala @@ -436,14 +436,8 @@ class DriverAndYarnResource( } def isModuleOperate(r: Resource): Boolean = { - if (this.isModuleOperate || r.isModuleOperate) { - true - } else if (this.yarnResource.queueName.equals(r.yarnResource.queueName)) { - logger.debug(s"Not module operate this:$this other:$r") - false - } else { - true - } + // todo consider optimization + false } def isModuleOperate: Boolean = { From 91aa53f3ef239c9fb79c325716c3a7bf0db02565 Mon Sep 17 00:00:00 2001 From: Casion Date: Wed, 22 Mar 2023 19:33:13 +0800 Subject: [PATCH 022/261] optimize scriptis code print (#103) --- .../entrance/restful/EntranceRestfulApi.java | 18 ------------------ .../impl/VarSubstitutionInterceptor.scala | 18 +++++++++++++++++- 2 files changed, 17 insertions(+), 19 deletions(-) diff --git a/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/restful/EntranceRestfulApi.java b/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/restful/EntranceRestfulApi.java index a5d3ace35f..eab67b2fe8 100644 --- a/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/restful/EntranceRestfulApi.java +++ b/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/restful/EntranceRestfulApi.java @@ -110,15 +110,6 @@ public Message execute(HttpServletRequest req, @RequestBody Map JobRequest jobReq = ((EntranceJob) job).getJobRequest(); Long jobReqId = jobReq.getId(); ModuleUserUtils.getOperationUser(req, "execute task,id: " + jobReqId); - pushLog( - LogUtils.generateInfo( - "You have submitted a new job, script code (after variable substitution) is"), - job); - pushLog( - "************************************SCRIPT CODE************************************", job); - pushLog(jobReq.getExecutionCode(), job); - pushLog( - "************************************SCRIPT CODE************************************", job); String execID = ZuulEntranceUtils.generateExecID( job.getId(), @@ -162,15 +153,6 @@ public Message submit(HttpServletRequest req, @RequestBody Map j JobRequest jobRequest = ((EntranceJob) job).getJobRequest(); Long jobReqId = jobRequest.getId(); ModuleUserUtils.getOperationUser(req, "submit jobReqId: " + jobReqId); - pushLog( - LogUtils.generateInfo( - "You have submitted a new job, script code (after variable substitution) is"), - job); - pushLog( - "************************************SCRIPT CODE************************************", job); - pushLog(jobRequest.getExecutionCode(), job); - pushLog( - "************************************SCRIPT CODE************************************", job); pushLog( LogUtils.generateInfo( "Your job is accepted, jobID is " diff --git a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/interceptor/impl/VarSubstitutionInterceptor.scala b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/interceptor/impl/VarSubstitutionInterceptor.scala index 0487a238cf..72d40305a6 100644 --- a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/interceptor/impl/VarSubstitutionInterceptor.scala +++ b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/interceptor/impl/VarSubstitutionInterceptor.scala @@ -41,10 +41,26 @@ class VarSubstitutionInterceptor extends EntranceInterceptor { LogUtils.generateInfo("Program is substituting variables for you") + "\n" ) val codeType = LabelUtil.getCodeType(jobRequest.getLabels) - jobRequest.setExecutionCode(CustomVariableUtils.replaceCustomVar(jobRequest, codeType)) + val realCode = CustomVariableUtils.replaceCustomVar(jobRequest, codeType) + jobRequest.setExecutionCode(realCode) logAppender.append( LogUtils.generateInfo("Variables substitution ended successfully") + "\n" ) + // print code after variables substitution + logAppender.append( + LogUtils.generateInfo( + "You have submitted a new job, script code (after variable substitution) is" + ) + "\n" + ); + logAppender.append( + "************************************SCRIPT CODE************************************" + "\n" + ) + logAppender.append(realCode); + logAppender.append("\n"); + logAppender.append( + "************************************SCRIPT CODE************************************" + "\n" + ); + jobRequest } { case e: VarSubstitutionException => From faa410f8c7a0270dce26ac06bb3ae6ba302d5d4e Mon Sep 17 00:00:00 2001 From: Casion Date: Wed, 22 Mar 2023 19:34:14 +0800 Subject: [PATCH 023/261] support spark.external.default.jars (#101) --- ...kSubmitProcessEngineConnLaunchBuilder.scala | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/launch/SparkSubmitProcessEngineConnLaunchBuilder.scala b/linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/launch/SparkSubmitProcessEngineConnLaunchBuilder.scala index f25a1d55b0..8b13e20ce1 100644 --- a/linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/launch/SparkSubmitProcessEngineConnLaunchBuilder.scala +++ b/linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/launch/SparkSubmitProcessEngineConnLaunchBuilder.scala @@ -18,6 +18,7 @@ package org.apache.linkis.engineplugin.spark.launch import org.apache.linkis.common.conf.CommonVars +import org.apache.linkis.common.utils.Logging import org.apache.linkis.engineplugin.spark.config.SparkConfiguration.{ ENGINE_JAR, SPARK_APP_NAME, @@ -42,7 +43,8 @@ import org.apache.commons.lang3.StringUtils import scala.collection.JavaConverters._ import scala.collection.mutable.ArrayBuffer -class SparkSubmitProcessEngineConnLaunchBuilder(builder: JavaProcessEngineConnLaunchBuilder) { +class SparkSubmitProcessEngineConnLaunchBuilder(builder: JavaProcessEngineConnLaunchBuilder) + extends Logging { def getCommands( engineConnBuildRequest: EngineConnBuildRequest, @@ -54,6 +56,10 @@ class SparkSubmitProcessEngineConnLaunchBuilder(builder: JavaProcessEngineConnLa val darResource = userEngineResource.getLockedResource.asInstanceOf[DriverAndYarnResource] val properties = engineConnBuildRequest.engineConnCreationDesc.properties + if (logger.isDebugEnabled()) { + logger.debug(s"spark engine with properties:$properties") + } + val className = getValueAndRemove(properties, "className", mainClass) val driverCores = getValueAndRemove(properties, LINKIS_SPARK_DRIVER_CORES) val driverMemory = getValueAndRemove(properties, LINKIS_SPARK_DRIVER_MEMORY) @@ -63,12 +69,13 @@ class SparkSubmitProcessEngineConnLaunchBuilder(builder: JavaProcessEngineConnLa val files = getValueAndRemove(properties, "files", "").split(",").filter(isNotBlankPath) val jars = new ArrayBuffer[String]() - jars ++= getValueAndRemove(properties, "jars", "").split(",").filter(isNotBlankPath) - jars ++= getValueAndRemove(properties, SPARK_DEFAULT_EXTERNAL_JARS_PATH) + // jars ++= getValueAndRemove(properties, "jars", "").split(",").filter(isNotBlankPath) + var externalJars = getValueAndRemove(properties, SPARK_DEFAULT_EXTERNAL_JARS_PATH) .split(",") .filter(x => { - isNotBlankPath(x) && (new java.io.File(x)).isFile + isNotBlankPath(x) }) + jars ++= externalJars val pyFiles = getValueAndRemove(properties, "py-files", "").split(",").filter(isNotBlankPath) val archives = getValueAndRemove(properties, "archives", "").split(",").filter(isNotBlankPath) @@ -208,6 +215,9 @@ class SparkSubmitProcessEngineConnLaunchBuilder(builder: JavaProcessEngineConnLa ): T = { val value = commonVars.getValue(properties) properties.remove(commonVars.key) + if (logger.isDebugEnabled()) { + logger.debug(s"getValueAndRemove with key:${commonVars.key},value:$value") + } value } From 10db0747273543a2269f69401886903a5dc133ed Mon Sep 17 00:00:00 2001 From: Casion Date: Wed, 22 Mar 2023 19:35:09 +0800 Subject: [PATCH 024/261] spark task add resource info log (#104) --- .../executor/SparkEngineConnExecutor.scala | 25 +++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/executor/SparkEngineConnExecutor.scala b/linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/executor/SparkEngineConnExecutor.scala index cb2f6187ed..837f5f9545 100644 --- a/linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/executor/SparkEngineConnExecutor.scala +++ b/linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/executor/SparkEngineConnExecutor.scala @@ -129,6 +129,31 @@ abstract class SparkEngineConnExecutor(val sc: SparkContext, id: Long) logger.info("Set jobGroup to " + jobGroup) sc.setJobGroup(jobGroup, _code, true) + // print job configuration + Utils.tryCatch({ + val executorNum: Int = sc.getConf.get("spark.executor.instances").toInt + val executorMem: Long = + ByteTimeUtils.byteStringAsGb(sc.getConf.get("spark.executor.memory")) * executorNum + val driverMem: Long = ByteTimeUtils.byteStringAsGb(sc.getConf.get("spark.driver.memory")) + val sparkExecutorCores = sc.getConf.get("spark.executor.cores", "2").toInt * executorNum + val sparkDriverCores = sc.getConf.get("spark.driver.cores", "1").toInt + val queue = sc.getConf.get("spark.yarn.queue") + + val sb = new StringBuilder + sb.append(s"spark.executor.instances=$executorNum\n") + sb.append(s"spark.executor.memory=${executorMem}G\n") + sb.append(s"spark.driver.memory=${driverMem}G\n") + sb.append(s"spark.executor.memory=${executorMem}G\n") + sb.append(s"spark.executor.cores=$sparkExecutorCores\n") + sb.append(s"spark.driver.cores=$sparkDriverCores\n") + sb.append(s"spark.yarn.queue=$queue\n") + engineExecutionContext.appendStdout( + LogUtils.generateInfo(s" Your spark job exec with configs:\n${sb.toString()}") + ) + })(t => { + logger.warn("Get actual used resource exception", t) + }) + val response = Utils.tryFinally(runCode(this, _code, engineExecutorContext, jobGroup)) { // Utils.tryAndWarn(this.engineExecutionContext.pushProgress(1, getProgressInfo(""))) jobGroup = null From fb5d4976c23c974158a4ad79a4a12eaa6525d46f Mon Sep 17 00:00:00 2001 From: huangKai-2323 <62878639+huangKai-2323@users.noreply.github.com> Date: Wed, 22 Mar 2023 20:04:15 +0800 Subject: [PATCH 025/261] =?UTF-8?q?=E3=80=901.1.10=E3=80=91Entry=20tenant?= =?UTF-8?q?=20label=20supports=20common=20configuration=20(#91)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../impl/TenantLabelSetUtils.scala | 5 ++ .../api/TenantConfigrationRestfulApi.java | 17 +++-- .../service/TenantConfigService.java | 2 +- .../service/impl/TenantConfigServiceImpl.java | 76 ++++++++++--------- 4 files changed, 54 insertions(+), 46 deletions(-) diff --git a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/interceptor/impl/TenantLabelSetUtils.scala b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/interceptor/impl/TenantLabelSetUtils.scala index 1deee62a72..49f44edbae 100644 --- a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/interceptor/impl/TenantLabelSetUtils.scala +++ b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/interceptor/impl/TenantLabelSetUtils.scala @@ -95,6 +95,11 @@ object TenantLabelSetUtils extends Logging { "*-" + LabelUtil.getUserCreatorLabel(jobRequest.getLabels).getCreator.toLowerCase() ) } + if (StringUtils.isBlank(tenant)) { + tenant = userCreatorTenantCache.get( + LabelUtil.getUserCreatorLabel(jobRequest.getLabels).getUser.toLowerCase() + "-*" + ) + } logger.info("get cache tenant:" + tenant + ",jobRequest:" + jobRequest.getId) // Add cached data if it is not empty if (StringUtils.isNotBlank(tenant)) { diff --git a/linkis-public-enhancements/linkis-configuration/src/main/java/org/apache/linkis/configuration/restful/api/TenantConfigrationRestfulApi.java b/linkis-public-enhancements/linkis-configuration/src/main/java/org/apache/linkis/configuration/restful/api/TenantConfigrationRestfulApi.java index e399849be3..4498fc3563 100644 --- a/linkis-public-enhancements/linkis-configuration/src/main/java/org/apache/linkis/configuration/restful/api/TenantConfigrationRestfulApi.java +++ b/linkis-public-enhancements/linkis-configuration/src/main/java/org/apache/linkis/configuration/restful/api/TenantConfigrationRestfulApi.java @@ -75,7 +75,7 @@ public Message createTenant(HttpServletRequest req, @RequestBody TenantVo tenant if (!Configuration.isAdmin(userName)) { return Message.error("Failed to create-tenant,msg: only administrators can configure"); } - if (tenantConfigService.userExists(tenantVo.getUser(), tenantVo.getCreator(), null)) { + if (tenantConfigService.isExist(tenantVo.getUser(), tenantVo.getCreator())) { throw new ConfigurationException("User-creator is existed"); } parameterVerification(tenantVo); @@ -226,8 +226,8 @@ public Message checkUserCreator( HttpServletRequest req, @RequestParam(value = "user", required = false) String user, @RequestParam(value = "creator", required = false) String creator, - @RequestParam(value = "tenantValue", required = false) String tenantValue) { - Boolean result = false; + @RequestParam(value = "id", required = false) String id) { + Boolean checkResult = false; try { // Parameter verification if (StringUtils.isBlank(creator)) { @@ -236,18 +236,19 @@ public Message checkUserCreator( if (StringUtils.isBlank(user)) { throw new ConfigurationException("User Name can't be empty "); } - if (creator.equals("*")) { - throw new ConfigurationException("Application Name can't be '*' "); - } String userName = ModuleUserUtils.getOperationUser(req, "checkUserCreator"); if (!Configuration.isAdmin(userName)) { return Message.error("Failed to check-user-creator,msg: only administrators can configure"); } - result = tenantConfigService.userExists(user, creator, tenantValue); + checkResult = tenantConfigService.isExist(user, creator); + // The incoming id represents the update. The update allows user-create exist + if (StringUtils.isNotBlank(id)) { + checkResult = !checkResult; + } } catch (ConfigurationException e) { return Message.error("Failed to check-user-creator,msg:" + e.getMessage()); } - return Message.ok().data("exist", result); + return Message.ok().data("exist", checkResult); } private void parameterVerification(TenantVo tenantVo) throws ConfigurationException { diff --git a/linkis-public-enhancements/linkis-configuration/src/main/java/org/apache/linkis/configuration/service/TenantConfigService.java b/linkis-public-enhancements/linkis-configuration/src/main/java/org/apache/linkis/configuration/service/TenantConfigService.java index 3d07ad6762..87b14a9c5e 100644 --- a/linkis-public-enhancements/linkis-configuration/src/main/java/org/apache/linkis/configuration/service/TenantConfigService.java +++ b/linkis-public-enhancements/linkis-configuration/src/main/java/org/apache/linkis/configuration/service/TenantConfigService.java @@ -33,7 +33,7 @@ Map queryTenantList( void createTenant(TenantVo tenantVo) throws ConfigurationException; - Boolean userExists(String user, String creator, String tenantValue) throws ConfigurationException; + Boolean isExist(String user, String creator) throws ConfigurationException; TenantVo queryTenant(String user, String creator); } diff --git a/linkis-public-enhancements/linkis-configuration/src/main/java/org/apache/linkis/configuration/service/impl/TenantConfigServiceImpl.java b/linkis-public-enhancements/linkis-configuration/src/main/java/org/apache/linkis/configuration/service/impl/TenantConfigServiceImpl.java index 8f1cd7f6ea..323d37ef12 100644 --- a/linkis-public-enhancements/linkis-configuration/src/main/java/org/apache/linkis/configuration/service/impl/TenantConfigServiceImpl.java +++ b/linkis-public-enhancements/linkis-configuration/src/main/java/org/apache/linkis/configuration/service/impl/TenantConfigServiceImpl.java @@ -105,7 +105,6 @@ public void updateTenant(TenantVo tenantVo) throws ConfigurationException { } dataProcessing(tenantVo); TenantVo tenantVoLowerCase = toLowerCase(tenantVo); - tenantVoLowerCase.setUpdateTime(new Date()); logger.info("updateTenant : {}", tenantVoLowerCase); userTenantMapper.updateTenant(tenantVoLowerCase); } @@ -119,55 +118,57 @@ public void updateTenant(TenantVo tenantVo) throws ConfigurationException { public void createTenant(TenantVo tenantVo) throws ConfigurationException { dataProcessing(tenantVo); TenantVo tenantVoLowerCase = toLowerCase(tenantVo); - tenantVoLowerCase.setUpdateTime(new Date()); tenantVoLowerCase.setCreateTime(new Date()); - logger.info("updateTenant : {}", tenantVoLowerCase); + logger.info("createTenant : {}", tenantVoLowerCase); userTenantMapper.createTenant(tenantVo); } private void dataProcessing(TenantVo tenantVo) throws ConfigurationException { - AtomicReference tenantResult = new AtomicReference<>(false); - // Obtain the tenant information of the ECM list - Map resultmap = null; - try { - resultmap = HttpsUtil.sendHttp(null, null); - logger.info("ResourceMonitor response {}:", resultmap); - } catch (IOException e) { - logger.warn("failed to get ecmResource data"); + if (!tenantVo.getCreator().equals("*")) { + AtomicReference tenantResult = new AtomicReference<>(false); + // Obtain the tenant information of the ECM list + Map ecmListResult = null; + try { + ecmListResult = HttpsUtil.sendHttp(null, null); + logger.info("Request ecm list response {}:", ecmListResult); + } catch (IOException e) { + logger.warn("failed to get ecmResource data"); + } + Map>> data = MapUtils.getMap(ecmListResult, "data"); + List> emNodeVoList = data.get("EMs"); + // Compare ECM list tenant labels for task + emNodeVoList.forEach( + ecmInfo -> { + List> labels = (List>) ecmInfo.get("labels"); + labels.stream() + .filter(labelmap -> labelmap.containsKey("tenant")) + .forEach( + map -> { + String tenant = map.get("tenant").toString().toLowerCase(); + if (tenant.equals(tenantVo.getTenantValue())) { + tenantResult.set(true); + } + }); + }); + // Compare the value of ecm tenant + if (!tenantResult.get()) + throw new ConfigurationException("The ECM with the corresponding label was not found"); + // The beginning of tenantValue needs to contain creator + String creator = tenantVo.getCreator().toLowerCase(); + String tenantValue = tenantVo.getTenantValue().toLowerCase().split("_")[0]; + if (!creator.equals(tenantValue)) + throw new ConfigurationException("tenantValue should contain creator first"); } - Map>> data = MapUtils.getMap(resultmap, "data"); - List> emNodeVoList = data.get("EMs"); - emNodeVoList.forEach( - ecmInfo -> { - List> labels = (List>) ecmInfo.get("labels"); - labels.stream() - .filter(labelmap -> labelmap.containsKey("tenant")) - .forEach( - map -> { - String tenant = map.get("tenant").toString().toLowerCase(); - if (tenant.equals(tenantVo.getTenantValue())) { - tenantResult.set(true); - } - }); - }); - // Compare the value of ecm tenant - if (!tenantResult.get()) - throw new ConfigurationException("The ECM with the corresponding label was not found"); - // The beginning of tenantValue needs to contain creator - String creator = tenantVo.getCreator().toLowerCase(); - String tenantValue = tenantVo.getTenantValue().toLowerCase().split("_")[0]; - if (!creator.equals(tenantValue)) - throw new ConfigurationException("tenantValue should contain creator first"); } @Override - public Boolean userExists(String user, String creator, String tenantValue) { - boolean result = true; + public Boolean isExist(String user, String creator) { + boolean result = false; Map resultMap = queryTenantList(user.toLowerCase(), creator.toLowerCase(), null, 1, 20); Object tenantList = resultMap.getOrDefault(JobRequestConstants.TOTAL_PAGE(), 0); int total = Integer.parseInt(tenantList.toString()); - if (total == 0) result = false; + if (total == 0) result = true; return result; } @@ -180,6 +181,7 @@ public TenantVo toLowerCase(TenantVo tenantVo) { tenantVo.setTenantValue(tenantVo.getTenantValue().toLowerCase()); tenantVo.setCreator(tenantVo.getCreator().toLowerCase()); tenantVo.setUser(tenantVo.getUser().toLowerCase()); + tenantVo.setUpdateTime(new Date()); return tenantVo; } } From eef38a569eabe7b4f7d846f7fd02066c4e04d6d3 Mon Sep 17 00:00:00 2001 From: Casion Date: Thu, 23 Mar 2023 01:22:37 +0800 Subject: [PATCH 026/261] fix csv export possible missing escape characters (#105) --- .../scala/org/apache/linkis/storage/csv/StorageCSVWriter.scala | 2 ++ 1 file changed, 2 insertions(+) diff --git a/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/csv/StorageCSVWriter.scala b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/csv/StorageCSVWriter.scala index f9b811b6a6..27a930a19b 100644 --- a/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/csv/StorageCSVWriter.scala +++ b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/csv/StorageCSVWriter.scala @@ -36,6 +36,8 @@ class StorageCSVWriter( with Logging { private val delimiter = separator match { + // Compatible with possible missing escape characters + case "t" => '\t' case separ if StringUtils.isNotEmpty(separ) => separ case _ => '\t' } From d8b17d1927acbdf0b7fd3e0b6e3d083e6970c3df Mon Sep 17 00:00:00 2001 From: binbincheng <106590848+binbincheng@users.noreply.github.com> Date: Thu, 23 Mar 2023 01:25:17 +0800 Subject: [PATCH 027/261] Improve exception information and add path information (#4351) Co-authored-by: casionone --- .../org/apache/linkis/storage/excel/XlsUtils.java | 1 + .../org/apache/linkis/storage/fs/FileSystem.java | 6 ++++++ .../linkis/storage/fs/impl/LocalFileSystem.java | 15 +++++++++++++-- .../resultset/DefaultResultSetFactory.scala | 2 +- .../storage/resultset/ResultSetReader.scala | 4 +++- .../storage/resultset/StorageResultSet.scala | 1 + .../resultset/StorageResultSetWriter.scala | 1 + 7 files changed, 26 insertions(+), 4 deletions(-) diff --git a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/excel/XlsUtils.java b/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/excel/XlsUtils.java index f3b8f44865..68469cb520 100644 --- a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/excel/XlsUtils.java +++ b/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/excel/XlsUtils.java @@ -61,6 +61,7 @@ public static String excelToCsv( throws Exception { String hdfsPath = "/tmp/" + StorageUtils.getJvmUser() + "/" + System.currentTimeMillis() + ".csv"; + LOG.info("The excel to csv with hdfs path:" + hdfsPath); ExcelXlsReader xlsReader = new ExcelXlsReader(); RowToCsvDeal rowToCsvDeal = new RowToCsvDeal(); OutputStream out = null; diff --git a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/fs/FileSystem.java b/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/fs/FileSystem.java index c19213a9cc..4bb7cb33b4 100644 --- a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/fs/FileSystem.java +++ b/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/fs/FileSystem.java @@ -24,8 +24,13 @@ import java.io.File; import java.io.IOException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + public abstract class FileSystem implements Fs { + private static final Logger LOG = LoggerFactory.getLogger(FileSystem.class); + protected String user; private String defaultFilePerm = "rwxr-----"; // 740 private String defaultFolderPerm = "rwxr-x---"; // 750 @@ -94,6 +99,7 @@ protected FsPath getParentPath(String path) { } else { parentPath = path.substring(0, path.lastIndexOf("/")); } + LOG.info("Get parent path:" + parentPath); return new FsPath(parentPath); } diff --git a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/fs/impl/LocalFileSystem.java b/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/fs/impl/LocalFileSystem.java index ce2ee43b7e..2df547f10e 100644 --- a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/fs/impl/LocalFileSystem.java +++ b/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/fs/impl/LocalFileSystem.java @@ -75,12 +75,14 @@ public String listRoot() throws IOException { @Override public long getTotalSpace(FsPath dest) throws IOException { String path = dest.getPath(); + LOG.info("Get total space with path:" + path); return new File(path).getTotalSpace(); } @Override public long getFreeSpace(FsPath dest) throws IOException { String path = dest.getPath(); + LOG.info("Get free space with path:" + path); return new File(path).getFreeSpace(); } @@ -117,6 +119,7 @@ public boolean setOwner(FsPath dest, String user, String group) throws IOExcepti @Override public boolean setOwner(FsPath dest, String user) throws IOException { + LOG.info("Set owner with path:" + dest.getPath() + "and user:" + user); if (!StorageUtils.isIOProxy()) { LOG.info("io not proxy, setOwner skip"); return true; @@ -133,6 +136,7 @@ public boolean setOwner(FsPath dest, String user) throws IOException { @Override public boolean setGroup(FsPath dest, String group) throws IOException { + LOG.info("Set group with path:" + dest.getPath() + "and group:" + user); if (!StorageUtils.isIOProxy()) { LOG.info("io not proxy, setGroup skip"); return true; @@ -155,6 +159,7 @@ public boolean mkdir(FsPath dest) throws IOException { @Override public boolean mkdirs(FsPath dest) throws IOException { String path = dest.getPath(); + LOG.info("Try to mkdirs with path:" + path); File file = new File(path); // Create parent directories one by one and set their permissions to rwxrwxrwx. Stack dirsToMake = new Stack(); @@ -182,6 +187,7 @@ public boolean mkdirs(FsPath dest) throws IOException { } public boolean canMkdir(FsPath destParentDir) throws IOException { + LOG.info("Try to check if the directory can be created with path:" + destParentDir.getPath()); if (!StorageUtils.isIOProxy()) { LOG.debug("io not proxy, not check owner, just check if have write permission "); return this.canWrite(destParentDir); @@ -203,6 +209,7 @@ public boolean canMkdir(FsPath destParentDir) throws IOException { @Override public boolean copy(String origin, String dest) throws IOException { File file = new File(dest); + LOG.info("Try to copy file from:" + origin + " to dest:" + dest); if (!isOwner(file.getParent())) { throw new IOException("you have on permission to create file " + dest); } @@ -225,6 +232,7 @@ public boolean copy(String origin, String dest) throws IOException { @Override public boolean setPermission(FsPath dest, String permission) throws IOException { + LOG.info("Try to set permission dest with path:" + dest.getPath()); if (!StorageUtils.isIOProxy()) { LOG.info("io not proxy, setPermission as parent."); try { @@ -251,6 +259,7 @@ public boolean setPermission(FsPath dest, String permission) throws IOException public FsPathListWithError listPathWithError(FsPath path) throws IOException { File file = new File(path.getPath()); File[] files = file.listFiles(); + LOG.info("Try to list path:" + path.getPath() + " with error msg"); if (files != null) { List rtn = new ArrayList(); String message = ""; @@ -294,6 +303,7 @@ public void init(Map properties) throws IOException { String groupInfo; try { groupInfo = Utils.exec(new String[] {"id", user}); + LOG.info("Get groupinfo:" + groupInfo + " with shell command: id " + user); } catch (RuntimeException e) { group = user; return; @@ -322,7 +332,7 @@ public FsPath get(String dest) throws IOException { } else { fsPath = new FsPath(dest); } - + LOG.info("Try to get FsPath with path:" + fsPath.getPath()); PosixFileAttributes attr = null; try { attr = Files.readAttributes(Paths.get(fsPath.getPath()), PosixFileAttributes.class); @@ -365,7 +375,7 @@ public OutputStream write(FsPath dest, boolean overwrite) throws IOException { @Override public boolean create(String dest) throws IOException { - + LOG.info("try to create file with path:" + dest); File file = new File(dest); if (!isOwner(file.getParent())) { throw new IOException("you have on permission to create file " + dest); @@ -391,6 +401,7 @@ public boolean create(String dest) throws IOException { public List list(FsPath path) throws IOException { File file = new File(path.getPath()); File[] files = file.listFiles(); + LOG.info("Try to get file list with path:" + path.getPath()); if (files != null) { List rtn = new ArrayList(); for (File f : files) { diff --git a/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/resultset/DefaultResultSetFactory.scala b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/resultset/DefaultResultSetFactory.scala index 38973ae1ab..d4836731db 100644 --- a/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/resultset/DefaultResultSetFactory.scala +++ b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/resultset/DefaultResultSetFactory.scala @@ -108,7 +108,7 @@ class DefaultResultSetFactory extends ResultSetFactory with Logging { if (StringUtils.isEmpty(resultSetType)) { throw new StorageWarnException( THE_FILE_IS_EMPTY.getErrorCode, - s"The file (${fsPath.getPath}) is empty(文件(${fsPath.getPath}) 为空)" + MessageFormat.format(THE_FILE_IS_EMPTY.getErrorDesc, fsPath.getPath) ) } Utils.tryQuietly(inputStream.close()) diff --git a/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/resultset/ResultSetReader.scala b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/resultset/ResultSetReader.scala index 663e379b5b..e61cf36b3d 100644 --- a/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/resultset/ResultSetReader.scala +++ b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/resultset/ResultSetReader.scala @@ -19,6 +19,7 @@ package org.apache.linkis.storage.resultset import org.apache.linkis.common.io.{FsPath, MetaData, Record} import org.apache.linkis.common.io.resultset.{ResultSet, ResultSetReader} +import org.apache.linkis.common.utils.Logging import org.apache.linkis.storage.FSFactory import org.apache.linkis.storage.errorcode.LinkisStorageErrorCodeSummary.TABLE_ARE_NOT_SUPPORTED import org.apache.linkis.storage.exception.StorageErrorException @@ -26,7 +27,7 @@ import org.apache.linkis.storage.resultset.table.{TableMetaData, TableRecord, Ta import java.io.InputStream -object ResultSetReader { +object ResultSetReader extends Logging { def getResultSetReader[K <: MetaData, V <: Record]( resultSet: ResultSet[K, V], @@ -83,6 +84,7 @@ object ResultSetReader { ) } val fs = FSFactory.getFs(resPath) + logger.info("Try to init Fs with path:" + resPath.getPath) fs.init(null) ResultSetReader.getResultSetReader(resultSet.asInstanceOf[TableResultSet], fs.read(resPath)) } diff --git a/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/resultset/StorageResultSet.scala b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/resultset/StorageResultSet.scala index fc303fbb5c..7b3aca62d9 100644 --- a/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/resultset/StorageResultSet.scala +++ b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/resultset/StorageResultSet.scala @@ -34,6 +34,7 @@ abstract class StorageResultSet[K <: MetaData, V <: Record] extends ResultSet[K, } else { parentDir.toPath + "/" + fileName + Dolphin.DOLPHIN_FILE_SUFFIX } + logger.info(s"Get result set path:${path}") new FsPath(path) } diff --git a/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/resultset/StorageResultSetWriter.scala b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/resultset/StorageResultSetWriter.scala index 9c7947272c..17106dca67 100644 --- a/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/resultset/StorageResultSetWriter.scala +++ b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/resultset/StorageResultSetWriter.scala @@ -86,6 +86,7 @@ class StorageResultSetWriter[K <: MetaData, V <: Record]( WRITER_LOCK_CREATE.synchronized { if (!fileCreated) { if (storePath != null && outputStream == null) { + logger.info(s"Try to create a new file:${storePath}, with proxy user:${proxyUser}") fs = FSFactory.getFsByProxyUser(storePath, proxyUser) fs.init(null) FileSystemUtils.createNewFile(storePath, proxyUser, true) From fcd3e20bfb27b87d92b6774b240fd48abdbaa77f Mon Sep 17 00:00:00 2001 From: casionone Date: Thu, 23 Mar 2023 02:43:37 +0800 Subject: [PATCH 028/261] revert of shell upload blocking --- .../main/scala/org/apache/linkis/common/utils/Utils.scala | 7 ------- 1 file changed, 7 deletions(-) diff --git a/linkis-commons/linkis-common/src/main/scala/org/apache/linkis/common/utils/Utils.scala b/linkis-commons/linkis-common/src/main/scala/org/apache/linkis/common/utils/Utils.scala index d469f60fba..80e3ff7e5e 100644 --- a/linkis-commons/linkis-common/src/main/scala/org/apache/linkis/common/utils/Utils.scala +++ b/linkis-commons/linkis-common/src/main/scala/org/apache/linkis/common/utils/Utils.scala @@ -304,13 +304,6 @@ object Utils extends Logging { pb.redirectInput(ProcessBuilder.Redirect.PIPE) val process = pb.start val log = new BufferedReader(new InputStreamReader(process.getInputStream)) - defaultScheduler.submit(new Runnable { - override def run(): Unit = { - while (process.isAlive) { - logger.info(log.readLine()) - } - } - }) val exitCode = if (maxWaitTime > 0) { val completed = process.waitFor(maxWaitTime, TimeUnit.MILLISECONDS) if (!completed) { From fca97ba8bf488c80ed21c070b5303b42cd7af73b Mon Sep 17 00:00:00 2001 From: aiceflower Date: Thu, 23 Mar 2023 10:41:04 +0800 Subject: [PATCH 029/261] move dir check to HDFSCacheLogWriter.init (#100) * move dir check to HDFSCacheLogWriter.init * format code * add lock * add lock * optimized code * Update HDFSCacheLogWriter.scala --------- Co-authored-by: aiceflower Co-authored-by: peacewong --- .../apache/linkis/entrance/parser/ParserUtils.java | 2 -- .../linkis/entrance/log/HDFSCacheLogWriter.scala | 14 +++++++++++--- .../linkis/entrance/utils/CommonLogPathUtils.scala | 8 ++++++-- 3 files changed, 17 insertions(+), 7 deletions(-) diff --git a/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/parser/ParserUtils.java b/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/parser/ParserUtils.java index 86af74d5c8..09622d55dc 100644 --- a/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/parser/ParserUtils.java +++ b/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/parser/ParserUtils.java @@ -19,7 +19,6 @@ import org.apache.linkis.common.io.FsPath; import org.apache.linkis.entrance.conf.EntranceConfiguration$; -import org.apache.linkis.entrance.utils.CommonLogPathUtils; import org.apache.linkis.governance.common.entity.job.JobRequest; import org.apache.linkis.manager.label.utils.LabelUtil; import org.apache.linkis.storage.utils.StorageUtils; @@ -69,7 +68,6 @@ public static void generateLogPath(JobRequest jobRequest, Map pa if (StorageUtils.HDFS().equals(lopPrefixPath.getFsType())) { String commonLogPath = logPathPrefix + "/" + "log" + "/" + dateString + "/" + creator; logPath = commonLogPath + "/" + umUser + "/" + jobRequest.getId() + ".log"; - CommonLogPathUtils.buildCommonPath(commonLogPath); } else { logPath = logPathPrefix diff --git a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/log/HDFSCacheLogWriter.scala b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/log/HDFSCacheLogWriter.scala index 24633dfbb2..dfd67bcb61 100644 --- a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/log/HDFSCacheLogWriter.scala +++ b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/log/HDFSCacheLogWriter.scala @@ -22,9 +22,10 @@ import org.apache.linkis.common.utils.Utils import org.apache.linkis.entrance.conf.EntranceConfiguration import org.apache.linkis.entrance.errorcode.EntranceErrorCodeSummary._ import org.apache.linkis.entrance.exception.EntranceErrorException +import org.apache.linkis.entrance.utils.CommonLogPathUtils import org.apache.linkis.storage.FSFactory import org.apache.linkis.storage.fs.FileSystem -import org.apache.linkis.storage.utils.FileSystemUtils +import org.apache.linkis.storage.utils.{FileSystemUtils, StorageUtils} import org.apache.commons.lang3.StringUtils import org.apache.hadoop.hdfs.client.HdfsDataOutputStream @@ -37,8 +38,9 @@ import java.util class HDFSCacheLogWriter(logPath: String, charset: String, sharedCache: Cache, user: String) extends LogWriter(charset) { - if (StringUtils.isBlank(logPath)) + if (StringUtils.isBlank(logPath)) { throw new EntranceErrorException(LOGPATH_NOT_NULL.getErrorCode, LOGPATH_NOT_NULL.getErrorDesc) + } protected var fileSystem = FSFactory.getFsByProxyUser(new FsPath(logPath), user).asInstanceOf[FileSystem] @@ -55,7 +57,13 @@ class HDFSCacheLogWriter(logPath: String, charset: String, sharedCache: Cache, u private def init(): Unit = { fileSystem.init(new util.HashMap[String, String]()) - FileSystemUtils.createNewFileWithFileSystem(fileSystem, new FsPath(logPath), user, true) + val logFsPath: FsPath = new FsPath(logPath) + Utils.tryCatch { + CommonLogPathUtils.buildCommonPath(logFsPath.getParent.getParent.getPath) + } { e: Throwable => + logger.warn("path check error. log path is: " + logPath, e) + } + FileSystemUtils.createNewFileWithFileSystem(fileSystem, logFsPath, user, true) } @throws[IOException] diff --git a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/utils/CommonLogPathUtils.scala b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/utils/CommonLogPathUtils.scala index 1311374fc1..f978be25d9 100644 --- a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/utils/CommonLogPathUtils.scala +++ b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/utils/CommonLogPathUtils.scala @@ -35,8 +35,12 @@ object CommonLogPathUtils { } val fsPath = new FsPath(realPath) if (!fileSystem.exists(fsPath)) { - FileSystemUtils.mkdirs(fileSystem, fsPath, StorageUtils.getJvmUser) - fileSystem.setPermission(fsPath, "770") + this.synchronized { + if (!fileSystem.exists(fsPath)) { + FileSystemUtils.mkdirs(fileSystem, fsPath, StorageUtils.getJvmUser) + fileSystem.setPermission(fsPath, "770") + } + } } Utils.tryQuietly(fileSystem.close()) } From 9d58e47b810d68d26b6899b63acb8d3612bc47f0 Mon Sep 17 00:00:00 2001 From: huangKai-2323 <62878639+huangKai-2323@users.noreply.github.com> Date: Thu, 23 Mar 2023 23:59:59 +0800 Subject: [PATCH 030/261] Dev 1.3.2 update tenant (#106) * update log * push code --------- Co-authored-by: Casion --- .../restful/api/TenantConfigrationRestfulApi.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/linkis-public-enhancements/linkis-configuration/src/main/java/org/apache/linkis/configuration/restful/api/TenantConfigrationRestfulApi.java b/linkis-public-enhancements/linkis-configuration/src/main/java/org/apache/linkis/configuration/restful/api/TenantConfigrationRestfulApi.java index 4498fc3563..9efbb6fd5a 100644 --- a/linkis-public-enhancements/linkis-configuration/src/main/java/org/apache/linkis/configuration/restful/api/TenantConfigrationRestfulApi.java +++ b/linkis-public-enhancements/linkis-configuration/src/main/java/org/apache/linkis/configuration/restful/api/TenantConfigrationRestfulApi.java @@ -75,7 +75,7 @@ public Message createTenant(HttpServletRequest req, @RequestBody TenantVo tenant if (!Configuration.isAdmin(userName)) { return Message.error("Failed to create-tenant,msg: only administrators can configure"); } - if (tenantConfigService.isExist(tenantVo.getUser(), tenantVo.getCreator())) { + if (!tenantConfigService.isExist(tenantVo.getUser(), tenantVo.getCreator())) { throw new ConfigurationException("User-creator is existed"); } parameterVerification(tenantVo); @@ -268,5 +268,8 @@ private void parameterVerification(TenantVo tenantVo) throws ConfigurationExcept if (StringUtils.isBlank(tenantVo.getTenantValue())) { throw new ConfigurationException("Tenant tag can't be empty "); } + if (tenantVo.getCreator().equals("*") && tenantVo.getUser().equals("*")) { + throw new ConfigurationException("User && Creator cannot be both *"); + } } } From dd10fbcce4ec2da9e50abec72cdce73d5151cf57 Mon Sep 17 00:00:00 2001 From: Casion Date: Fri, 24 Mar 2023 00:00:14 +0800 Subject: [PATCH 031/261] Fix custom variable settings (#107) * fix custom variable settings * code format --- .../script/writer/StorageScriptFsWriter.scala | 4 +- .../linkis/storage/source/FileSplit.scala | 6 +- .../writer/StorageScriptFsWriterTest.java | 87 +++++++++++++++++++ .../src/test/resources/scritpis-test2.sql | 9 ++ pom.xml | 2 +- 5 files changed, 99 insertions(+), 9 deletions(-) create mode 100644 linkis-commons/linkis-storage/src/test/resources/scritpis-test2.sql diff --git a/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/script/writer/StorageScriptFsWriter.scala b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/script/writer/StorageScriptFsWriter.scala index cdb9186da4..f8d4518c1d 100644 --- a/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/script/writer/StorageScriptFsWriter.scala +++ b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/script/writer/StorageScriptFsWriter.scala @@ -51,9 +51,7 @@ class StorageScriptFsWriter( .map(compaction.compact) .foreach(metadataLine.add) // add annotition symbol - if (metadataLine.size() > 0) { - metadataLine.add(compaction.getAnnotationSymbol()) - } + metadataLine.add(compaction.getAnnotationSymbol()) if (outputStream != null) { IOUtils.writeLines(metadataLine, "\n", outputStream, charset) } else { diff --git a/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/source/FileSplit.scala b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/source/FileSplit.scala index 467fbca335..3839800f12 100644 --- a/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/source/FileSplit.scala +++ b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/source/FileSplit.scala @@ -80,11 +80,7 @@ class FileSplit( var needRemoveFlag = false if (hasRemovedFlag == false && fsReader.isInstanceOf[StorageScriptFsReader]) { val parser = fsReader.asInstanceOf[StorageScriptFsReader].getScriptParser() - val meta = metaData.asInstanceOf[ScriptMetaData].getMetaData - if ( - meta != null && meta.length > 0 - && parser != null && parser.getAnnotationSymbol().equals(record.toString) - ) { + if (parser != null && parser.getAnnotationSymbol().equals(record.toString)) { needRemoveFlag = true hasRemovedFlag = true } diff --git a/linkis-commons/linkis-storage/src/test/java/org/apache/linkis/storage/script/writer/StorageScriptFsWriterTest.java b/linkis-commons/linkis-storage/src/test/java/org/apache/linkis/storage/script/writer/StorageScriptFsWriterTest.java index 60a593665c..523402c2ef 100644 --- a/linkis-commons/linkis-storage/src/test/java/org/apache/linkis/storage/script/writer/StorageScriptFsWriterTest.java +++ b/linkis-commons/linkis-storage/src/test/java/org/apache/linkis/storage/script/writer/StorageScriptFsWriterTest.java @@ -98,6 +98,43 @@ class StorageScriptFsWriterTest { String fileName = "229cf765-6839-4c82-829d-1907c2ccf668.sql"; + String scriptContent2 = + "" + + "select ${qq};\n" + + "--@set test=123\n" + + "select ${test};\n" + + "--@set qq=222\n" + + "select ${qq};\n" + + "--\n" + + "--\n" + + "select 1;"; + String metaData2 = ""; + + String resultMetaData2 = "{}"; + String resultString2 = + "" + + "--\n" + + "select ${qq};\n" + + "--@set test=123\n" + + "select ${test};\n" + + "--@set qq=222\n" + + "select ${qq};\n" + + "--\n" + + "--\n" + + "select 1;"; + + Map params2 = new HashMap<>(); + + { + try { + params = new ObjectMapper().readValue(metaData, HashMap.class); + } catch (JsonProcessingException e) { + e.printStackTrace(); + } + } + + String fileName2 = "229cf765-6839-4c82-829d-1907c2ccf669.sql"; + // return Message.ok().data("resourceId", resourceId).data("version", version); @Test @@ -127,6 +164,34 @@ void TestSave() { } } + @Test + void TestSave2() { + + ScriptFsWriter writer = + StorageScriptFsWriter.getScriptFsWriter(new FsPath(fileName2), "UTF-8", null); + Variable[] v = VariableParser.getVariables(params2); + List variableList = + Arrays.stream(v) + .filter(var -> !StringUtils.isEmpty(var.value())) + .collect(Collectors.toList()); + try { + + MetaData getMetaData = new ScriptMetaData(variableList.toArray(new Variable[0])); + writer.addMetaData(getMetaData); + writer.addRecord(new ScriptRecord(scriptContent2)); + InputStream inputStream = writer.getInputStream(); + + String text = + new BufferedReader(new InputStreamReader(inputStream, StandardCharsets.UTF_8)) + .lines() + .collect(Collectors.joining("\n")); + Assertions.assertEquals(text, resultString2); + + } catch (Exception e) { + e.printStackTrace(); + } + } + @Test void TestOpen() throws FileNotFoundException { @@ -148,4 +213,26 @@ void TestOpen() throws FileNotFoundException { Assertions.assertEquals(metadataRes, resultMetaData); } + + @Test + void TestOpen2() throws FileNotFoundException { + + // /api/rest_j/v1/filesystem/openScriptFromBML?fileName=229cf765-6839-4c82-829d-1907c2ccf668.sql&resourceId=c9755cad-619b-4c1c-9204-cc4bb9836194&version=v000008&creator=&projectName=test1122a1 + + String filePath = this.getClass().getResource("/scritpis-test2.sql").getFile().toString(); + + File file = new File(filePath); + + InputStream inputStream = new FileInputStream(file); + + FileSource fileSource = FileSource$.MODULE$.create(new FsPath(fileName2), inputStream); + Pair> collect = fileSource.collect()[0]; + + String scriptRes = collect.getSecond().get(0)[0]; + String metadataRes = new Gson().toJson(collect.getFirst()); + + Assertions.assertEquals(scriptContent2 + "\n", scriptRes); + + Assertions.assertEquals(metadataRes, resultMetaData2); + } } diff --git a/linkis-commons/linkis-storage/src/test/resources/scritpis-test2.sql b/linkis-commons/linkis-storage/src/test/resources/scritpis-test2.sql new file mode 100644 index 0000000000..8f4a6defbe --- /dev/null +++ b/linkis-commons/linkis-storage/src/test/resources/scritpis-test2.sql @@ -0,0 +1,9 @@ +-- +select ${qq}; +--@set test=123 +select ${test}; +--@set qq=222 +select ${qq}; +-- +-- +select 1; \ No newline at end of file diff --git a/pom.xml b/pom.xml index 0eaa57f1f5..fe5de420b1 100644 --- a/pom.xml +++ b/pom.xml @@ -1184,7 +1184,7 @@ compiler/** **/generated/** **/scritpis-test.sql - + **/scritpis-test2.sql From 1579ca1ff9ea5c18fc24a2f92e3f1f738a8a0d3f Mon Sep 17 00:00:00 2001 From: Casion Date: Fri, 24 Mar 2023 00:00:26 +0800 Subject: [PATCH 032/261] engine bml resource compatible prefix v format (#108) * engine bml resource compatible prefix v format * code format --- ...linkis-engineconn-plugin-core-errorcode.md | 2 +- .../server/security/SecurityFilter.scala | 6 +++- .../EngineconnCoreErrorCodeSummary.java | 2 +- .../conf/EngineConnPluginConfiguration.scala | 4 +++ ...stractEngineConnBmlResourceGenerator.scala | 36 ++++++++++++++++--- .../DefaultEngineConnResourceService.scala | 13 +++++-- .../service/impl/DefaultResourceManager.scala | 2 +- .../src/apps/linkis/i18n/common/en.json | 2 +- 8 files changed, 54 insertions(+), 13 deletions(-) diff --git a/docs/errorcode/linkis-engineconn-plugin-core-errorcode.md b/docs/errorcode/linkis-engineconn-plugin-core-errorcode.md index a929d062d2..0fc2521a76 100644 --- a/docs/errorcode/linkis-engineconn-plugin-core-errorcode.md +++ b/docs/errorcode/linkis-engineconn-plugin-core-errorcode.md @@ -3,7 +3,7 @@ | module name(模块名) | error code(错误码) | describe(描述) |enumeration name(枚举)| Exception Class(类名)| | -------- | -------- | ----- |-----|-----| |linkis-engineconn-plugin-core |10001|Failed to createEngineConnLaunchRequest(创建 EngineConnLaunchRequest失败)|FAILED_CREATE_ELR|EngineconnCoreErrorCodeSummary| -|linkis-engineconn-plugin-core |10001|The engine plug-in material is abnormal, please check whether the material is uploaded successfully(引擎插件物料异常，请检查物料是否上传成功)|EN_PLUGIN_MATERIAL_SOURCE_EXCEPTION|EngineconnCoreErrorCodeSummary| +|linkis-engineconn-plugin-core |10001|The engine plugin material is abnormal, please check whether the material is uploaded successfully(引擎插件物料异常，请检查物料是否上传成功)|EN_PLUGIN_MATERIAL_SOURCE_EXCEPTION|EngineconnCoreErrorCodeSummary| |linkis-engineconn-plugin-core |10001|EngineTypeLabel are requested(需要参数 EngineTypeLabel)|ETL_REQUESTED|EngineconnCoreErrorCodeSummary| |linkis-engineconn-plugin-core |20000|Cannot instance EngineConnExecution(无法实例化 EngineConnExecution)|CANNOT_INSTANCE_ECE|EngineconnCoreErrorCodeSummary| |linkis-engineconn-plugin-core |20000|Cannot find default ExecutorFactory(找不到默认的 ExecutorFactory)|CANNOT_DEFAULT_EF|EngineconnCoreErrorCodeSummary| diff --git a/linkis-commons/linkis-module/src/main/scala/org/apache/linkis/server/security/SecurityFilter.scala b/linkis-commons/linkis-module/src/main/scala/org/apache/linkis/server/security/SecurityFilter.scala index 411bda4820..5cc796d23e 100644 --- a/linkis-commons/linkis-module/src/main/scala/org/apache/linkis/server/security/SecurityFilter.scala +++ b/linkis-commons/linkis-module/src/main/scala/org/apache/linkis/server/security/SecurityFilter.scala @@ -199,7 +199,11 @@ object SecurityFilter { def getLoginUsername(req: HttpServletRequest): String = { if (Configuration.IS_TEST_MODE.getValue) { - ServerConfiguration.BDP_TEST_USER.getValue; + val testUser = ServerConfiguration.BDP_TEST_USER.getValue + if (StringUtils.isBlank(testUser)) { + throw new IllegalUserTicketException("Need to set test user when enable test module") + } + testUser } else { getLoginUser(req).getOrElse( throw new IllegalUserTicketException(ILLEGAL_USER_TOKEN.getErrorDesc) diff --git a/linkis-computation-governance/linkis-engineconn/linkis-engineconn-plugin-core/src/main/java/org/apache/linkis/manager/engineplugin/errorcode/EngineconnCoreErrorCodeSummary.java b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-plugin-core/src/main/java/org/apache/linkis/manager/engineplugin/errorcode/EngineconnCoreErrorCodeSummary.java index 1685f4b652..f29b2a6ed8 100644 --- a/linkis-computation-governance/linkis-engineconn/linkis-engineconn-plugin-core/src/main/java/org/apache/linkis/manager/engineplugin/errorcode/EngineconnCoreErrorCodeSummary.java +++ b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-plugin-core/src/main/java/org/apache/linkis/manager/engineplugin/errorcode/EngineconnCoreErrorCodeSummary.java @@ -23,7 +23,7 @@ public enum EngineconnCoreErrorCodeSummary implements LinkisErrorCode { FAILED_CREATE_ELR(10001, "Failed to createEngineConnLaunchRequest(创建 EngineConnLaunchRequest失败)"), EN_PLUGIN_MATERIAL_SOURCE_EXCEPTION( 10001, - "The engine plug-in material is abnormal, please check whether the material is uploaded successfully(引擎插件物料异常，请检查物料是否上传成功)"), + "The engine plugin material is abnormal, please check whether the material is uploaded successfully(引擎插件物料异常，请检查物料是否上传成功)"), ETL_REQUESTED(10001, "EngineTypeLabel are requested(需要参数 EngineTypeLabel)"), CANNOT_INSTANCE_ECE(20000, "Cannot instance EngineConnExecution(无法实例化 EngineConnExecution)"), diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/engineplugin/server/conf/EngineConnPluginConfiguration.scala b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/engineplugin/server/conf/EngineConnPluginConfiguration.scala index de1add173a..9a16d9b9e6 100644 --- a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/engineplugin/server/conf/EngineConnPluginConfiguration.scala +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/engineplugin/server/conf/EngineConnPluginConfiguration.scala @@ -35,4 +35,8 @@ object EngineConnPluginConfiguration { val ENABLED_BML_UPLOAD_FAILED_EXIT: CommonVars[Boolean] = CommonVars("wds.linkis.engineconn.bml.upload.failed.enable", true) + // for third party eg appconn/datax, if all update, can set to false then to remove + val EC_BML_VERSION_MAY_WITH_PREFIX_V: CommonVars[Boolean] = + CommonVars("linkis.engineconn.bml.version.may.with.prefix", true) + } diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/engineplugin/server/localize/AbstractEngineConnBmlResourceGenerator.scala b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/engineplugin/server/localize/AbstractEngineConnBmlResourceGenerator.scala index fce3508c31..6f23b5eec2 100644 --- a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/engineplugin/server/localize/AbstractEngineConnBmlResourceGenerator.scala +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/engineplugin/server/localize/AbstractEngineConnBmlResourceGenerator.scala @@ -17,6 +17,7 @@ package org.apache.linkis.engineplugin.server.localize +import org.apache.linkis.engineplugin.server.conf.EngineConnPluginConfiguration import org.apache.linkis.engineplugin.server.conf.EngineConnPluginConfiguration.ENGINE_CONN_HOME import org.apache.linkis.engineplugin.server.localize.EngineConnBmlResourceGenerator.NO_VERSION_MARK import org.apache.linkis.manager.engineplugin.common.exception.EngineConnPluginErrorException @@ -53,13 +54,38 @@ abstract class AbstractEngineConnBmlResourceGenerator extends EngineConnBmlResou val engineConnPackageHome = Paths.get(engineConnDistHome, version).toFile.getPath logger.info("getEngineConnDistHome, engineConnPackageHome path:" + engineConnPackageHome) val engineConnPackageHomeFile = new File(engineConnPackageHome) + + // 兼容老版本 if (!engineConnPackageHomeFile.exists()) { - throw new EngineConnPluginErrorException( - ENGINE_VERSION_NOT_FOUND.getErrorCode, - MessageFormat.format(ENGINE_VERSION_NOT_FOUND.getErrorDesc, version, engineConnType) - ) + if ( + !version.startsWith( + "v" + ) && EngineConnPluginConfiguration.EC_BML_VERSION_MAY_WITH_PREFIX_V.getValue + ) { + val versionOld = "v" + version + val engineConnPackageHomeOld = Paths.get(engineConnDistHome, versionOld).toFile.getPath + logger.info( + "try to getEngineConnDistHome with prefix v, engineConnPackageHome path:" + engineConnPackageHomeOld + ) + val engineConnPackageHomeFileOld = new File(engineConnPackageHomeOld) + if (!engineConnPackageHomeFileOld.exists()) { + throw new EngineConnPluginErrorException( + ENGINE_VERSION_NOT_FOUND.getErrorCode, + MessageFormat.format(ENGINE_VERSION_NOT_FOUND.getErrorDesc, version, engineConnType) + ) + } else { + engineConnPackageHomeOld + } + } else { + throw new EngineConnPluginErrorException( + ENGINE_VERSION_NOT_FOUND.getErrorCode, + MessageFormat.format(ENGINE_VERSION_NOT_FOUND.getErrorDesc, version, engineConnType) + ) + } + } else { + engineConnPackageHome } - engineConnPackageHome + } private def checkEngineConnDistHome(engineConnPackageHomePath: String): Unit = { diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/engineplugin/server/service/DefaultEngineConnResourceService.scala b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/engineplugin/server/service/DefaultEngineConnResourceService.scala index 3ddfdc7bbe..55f3b2568a 100644 --- a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/engineplugin/server/service/DefaultEngineConnResourceService.scala +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/engineplugin/server/service/DefaultEngineConnResourceService.scala @@ -201,8 +201,6 @@ class DefaultEngineConnResourceService extends EngineConnResourceService with Lo engineConnBmlResource.setFileName(localizeResource.fileName) engineConnBmlResource.setFileSize(localizeResource.fileSize) engineConnBmlResource.setLastModified(localizeResource.lastModified) - if (version.startsWith("v")) engineConnBmlResource.setVersion(version.substring(1)) - else engineConnBmlResource.setVersion(version) engineConnBmlResource.setVersion(version) engineConnBmlResourceDao.save(engineConnBmlResource) } else { @@ -241,9 +239,18 @@ class DefaultEngineConnResourceService extends EngineConnResourceService with Lo ): EngineConnResource = { val engineConnType = engineConnBMLResourceRequest.getEngineConnType val version = engineConnBMLResourceRequest.getVersion - val engineConnBmlResources = asScalaBufferConverter( + var engineConnBmlResources = asScalaBufferConverter( engineConnBmlResourceDao.getAllEngineConnBmlResource(engineConnType, version) ) + if ( + engineConnBmlResources.asScala.size == 0 && EngineConnPluginConfiguration.EC_BML_VERSION_MAY_WITH_PREFIX_V.getValue + ) { + logger.info("Try to get engine conn bml resource with prefex v") + engineConnBmlResources = asScalaBufferConverter( + engineConnBmlResourceDao.getAllEngineConnBmlResource(engineConnType, "v" + version) + ) + } + val confBmlResourceMap = engineConnBmlResources.asScala .find(_.getFileName == LaunchConstants.ENGINE_CONN_CONF_DIR_NAME + ".zip") .map(parseToBmlResource) diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/rm/service/impl/DefaultResourceManager.scala b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/rm/service/impl/DefaultResourceManager.scala index f1284268b3..6145f8bc0b 100644 --- a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/rm/service/impl/DefaultResourceManager.scala +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/rm/service/impl/DefaultResourceManager.scala @@ -710,7 +710,7 @@ class DefaultResourceManager extends ResourceManager with Logging with Initializ var heartbeatMsgMetrics = "" Utils.tryAndWarn { val oldMetrics = nodeMetricManagerPersistence.getNodeMetrics(ecNode) - if (StringUtils.isNotBlank(oldMetrics.getHeartBeatMsg)) { + if (oldMetrics != null && StringUtils.isNotBlank(oldMetrics.getHeartBeatMsg)) { heartbeatMsgMetrics = oldMetrics.getHeartBeatMsg } } diff --git a/linkis-web/src/apps/linkis/i18n/common/en.json b/linkis-web/src/apps/linkis/i18n/common/en.json index c6230555fb..845b6fe03a 100644 --- a/linkis-web/src/apps/linkis/i18n/common/en.json +++ b/linkis-web/src/apps/linkis/i18n/common/en.json @@ -629,7 +629,7 @@ "versionList": "Version List", "rollback": "Rollback", "checkEngineConnTypeAndVersion": "Please select the engine type and version", - "upload": "Please click the button to upload the engine plug-in" + "upload": "Please click the button to upload the engine plugin" } } } From 6901b5b7b7c8c21c762578d806ba4670d66abf6b Mon Sep 17 00:00:00 2001 From: peacewong Date: Fri, 24 Mar 2023 16:22:58 +0800 Subject: [PATCH 033/261] Revert "move dir check to HDFSCacheLogWriter.init (#100)" (#109) This reverts commit fca97ba8bf488c80ed21c070b5303b42cd7af73b. --- .../apache/linkis/entrance/parser/ParserUtils.java | 2 ++ .../linkis/entrance/log/HDFSCacheLogWriter.scala | 14 +++----------- .../linkis/entrance/utils/CommonLogPathUtils.scala | 8 ++------ 3 files changed, 7 insertions(+), 17 deletions(-) diff --git a/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/parser/ParserUtils.java b/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/parser/ParserUtils.java index 09622d55dc..86af74d5c8 100644 --- a/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/parser/ParserUtils.java +++ b/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/parser/ParserUtils.java @@ -19,6 +19,7 @@ import org.apache.linkis.common.io.FsPath; import org.apache.linkis.entrance.conf.EntranceConfiguration$; +import org.apache.linkis.entrance.utils.CommonLogPathUtils; import org.apache.linkis.governance.common.entity.job.JobRequest; import org.apache.linkis.manager.label.utils.LabelUtil; import org.apache.linkis.storage.utils.StorageUtils; @@ -68,6 +69,7 @@ public static void generateLogPath(JobRequest jobRequest, Map pa if (StorageUtils.HDFS().equals(lopPrefixPath.getFsType())) { String commonLogPath = logPathPrefix + "/" + "log" + "/" + dateString + "/" + creator; logPath = commonLogPath + "/" + umUser + "/" + jobRequest.getId() + ".log"; + CommonLogPathUtils.buildCommonPath(commonLogPath); } else { logPath = logPathPrefix diff --git a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/log/HDFSCacheLogWriter.scala b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/log/HDFSCacheLogWriter.scala index dfd67bcb61..24633dfbb2 100644 --- a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/log/HDFSCacheLogWriter.scala +++ b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/log/HDFSCacheLogWriter.scala @@ -22,10 +22,9 @@ import org.apache.linkis.common.utils.Utils import org.apache.linkis.entrance.conf.EntranceConfiguration import org.apache.linkis.entrance.errorcode.EntranceErrorCodeSummary._ import org.apache.linkis.entrance.exception.EntranceErrorException -import org.apache.linkis.entrance.utils.CommonLogPathUtils import org.apache.linkis.storage.FSFactory import org.apache.linkis.storage.fs.FileSystem -import org.apache.linkis.storage.utils.{FileSystemUtils, StorageUtils} +import org.apache.linkis.storage.utils.FileSystemUtils import org.apache.commons.lang3.StringUtils import org.apache.hadoop.hdfs.client.HdfsDataOutputStream @@ -38,9 +37,8 @@ import java.util class HDFSCacheLogWriter(logPath: String, charset: String, sharedCache: Cache, user: String) extends LogWriter(charset) { - if (StringUtils.isBlank(logPath)) { + if (StringUtils.isBlank(logPath)) throw new EntranceErrorException(LOGPATH_NOT_NULL.getErrorCode, LOGPATH_NOT_NULL.getErrorDesc) - } protected var fileSystem = FSFactory.getFsByProxyUser(new FsPath(logPath), user).asInstanceOf[FileSystem] @@ -57,13 +55,7 @@ class HDFSCacheLogWriter(logPath: String, charset: String, sharedCache: Cache, u private def init(): Unit = { fileSystem.init(new util.HashMap[String, String]()) - val logFsPath: FsPath = new FsPath(logPath) - Utils.tryCatch { - CommonLogPathUtils.buildCommonPath(logFsPath.getParent.getParent.getPath) - } { e: Throwable => - logger.warn("path check error. log path is: " + logPath, e) - } - FileSystemUtils.createNewFileWithFileSystem(fileSystem, logFsPath, user, true) + FileSystemUtils.createNewFileWithFileSystem(fileSystem, new FsPath(logPath), user, true) } @throws[IOException] diff --git a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/utils/CommonLogPathUtils.scala b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/utils/CommonLogPathUtils.scala index f978be25d9..1311374fc1 100644 --- a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/utils/CommonLogPathUtils.scala +++ b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/utils/CommonLogPathUtils.scala @@ -35,12 +35,8 @@ object CommonLogPathUtils { } val fsPath = new FsPath(realPath) if (!fileSystem.exists(fsPath)) { - this.synchronized { - if (!fileSystem.exists(fsPath)) { - FileSystemUtils.mkdirs(fileSystem, fsPath, StorageUtils.getJvmUser) - fileSystem.setPermission(fsPath, "770") - } - } + FileSystemUtils.mkdirs(fileSystem, fsPath, StorageUtils.getJvmUser) + fileSystem.setPermission(fsPath, "770") } Utils.tryQuietly(fileSystem.close()) } From 8f00cbb68a9a0da4fa51b3c20a8368ec14688b96 Mon Sep 17 00:00:00 2001 From: peacewong Date: Fri, 24 Mar 2023 16:47:33 +0800 Subject: [PATCH 034/261] Revert "fix shell upload blocking (#95)" (#110) This reverts commit 2d89558813f11a9da135e7a06b3ccdbe8b3a7140. From 0281513b3833c167ce937a936e7145772a640ff8 Mon Sep 17 00:00:00 2001 From: peacewong Date: Fri, 24 Mar 2023 23:18:55 +0800 Subject: [PATCH 035/261] Fix unlock --- .../executor/lock/EngineConnTimedLock.scala | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/org/apache/linkis/engineconn/acessible/executor/lock/EngineConnTimedLock.scala b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/org/apache/linkis/engineconn/acessible/executor/lock/EngineConnTimedLock.scala index 26a9203795..f8bc353a16 100644 --- a/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/org/apache/linkis/engineconn/acessible/executor/lock/EngineConnTimedLock.scala +++ b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/org/apache/linkis/engineconn/acessible/executor/lock/EngineConnTimedLock.scala @@ -21,13 +21,9 @@ import org.apache.linkis.common.utils.Logging import org.apache.linkis.engineconn.acessible.executor.conf.AccessibleExecutorConfiguration import org.apache.linkis.engineconn.acessible.executor.entity.AccessibleExecutor import org.apache.linkis.engineconn.acessible.executor.listener.ExecutorStatusListener -import org.apache.linkis.engineconn.acessible.executor.listener.event.{ - ExecutorCompletedEvent, - ExecutorCreateEvent, - ExecutorStatusChangedEvent, - ExecutorUnLockEvent -} +import org.apache.linkis.engineconn.acessible.executor.listener.event.{ExecutorCompletedEvent, ExecutorCreateEvent, ExecutorStatusChangedEvent, ExecutorUnLockEvent} import org.apache.linkis.engineconn.core.executor.ExecutorManager +import org.apache.linkis.engineconn.executor.entity.SensibleExecutor import org.apache.linkis.engineconn.executor.listener.ExecutorListenerBusContext import org.apache.linkis.manager.common.entity.enumeration.NodeStatus @@ -169,9 +165,14 @@ class EngineConnTimedLock(private var timeout: Long) } private def unlockCallback(lockStr: String): Unit = { - /* if (null != lockedBy) { - lockedBy.transition(NodeStatus.Unlock) - } */ + val nodeStatus = ExecutorManager.getInstance.getReportExecutor match { + case sensibleExecutor: SensibleExecutor => + sensibleExecutor.getStatus + case _ => NodeStatus.Idle + } + if (NodeStatus.isCompleted(nodeStatus)) { + logger.info("The node({}) is already in the completed state, and the unlocking is invalid", nodeStatus.toString) + } val executors = ExecutorManager.getInstance.getExecutors.filter(executor => null != executor && !executor.isClosed ) From 8f792d51c625528de1b3380aed0b2d02f0cd490d Mon Sep 17 00:00:00 2001 From: peacewong Date: Sun, 26 Mar 2023 22:11:21 +0800 Subject: [PATCH 036/261] Fix unlock (#111) --- .../executor/lock/EngineConnTimedLock.scala | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/org/apache/linkis/engineconn/acessible/executor/lock/EngineConnTimedLock.scala b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/org/apache/linkis/engineconn/acessible/executor/lock/EngineConnTimedLock.scala index 26a9203795..f8bc353a16 100644 --- a/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/org/apache/linkis/engineconn/acessible/executor/lock/EngineConnTimedLock.scala +++ b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/org/apache/linkis/engineconn/acessible/executor/lock/EngineConnTimedLock.scala @@ -21,13 +21,9 @@ import org.apache.linkis.common.utils.Logging import org.apache.linkis.engineconn.acessible.executor.conf.AccessibleExecutorConfiguration import org.apache.linkis.engineconn.acessible.executor.entity.AccessibleExecutor import org.apache.linkis.engineconn.acessible.executor.listener.ExecutorStatusListener -import org.apache.linkis.engineconn.acessible.executor.listener.event.{ - ExecutorCompletedEvent, - ExecutorCreateEvent, - ExecutorStatusChangedEvent, - ExecutorUnLockEvent -} +import org.apache.linkis.engineconn.acessible.executor.listener.event.{ExecutorCompletedEvent, ExecutorCreateEvent, ExecutorStatusChangedEvent, ExecutorUnLockEvent} import org.apache.linkis.engineconn.core.executor.ExecutorManager +import org.apache.linkis.engineconn.executor.entity.SensibleExecutor import org.apache.linkis.engineconn.executor.listener.ExecutorListenerBusContext import org.apache.linkis.manager.common.entity.enumeration.NodeStatus @@ -169,9 +165,14 @@ class EngineConnTimedLock(private var timeout: Long) } private def unlockCallback(lockStr: String): Unit = { - /* if (null != lockedBy) { - lockedBy.transition(NodeStatus.Unlock) - } */ + val nodeStatus = ExecutorManager.getInstance.getReportExecutor match { + case sensibleExecutor: SensibleExecutor => + sensibleExecutor.getStatus + case _ => NodeStatus.Idle + } + if (NodeStatus.isCompleted(nodeStatus)) { + logger.info("The node({}) is already in the completed state, and the unlocking is invalid", nodeStatus.toString) + } val executors = ExecutorManager.getInstance.getExecutors.filter(executor => null != executor && !executor.isClosed ) From e29390e4bedf15dbc1b7f6391165926c8a27d084 Mon Sep 17 00:00:00 2001 From: Casion Date: Sun, 26 Mar 2023 22:11:54 +0800 Subject: [PATCH 037/261] fix NPE bug (#112) fix variable identification problem eg -----@set x=y--- Optimize spark log to print repeatedly --- .../linkis/common/utils/VariableUtils.scala | 16 ++--- .../EngineconnCoreErrorCodeSummary.java | 2 +- .../entrance/EntranceWebSocketService.scala | 12 ---- .../interceptor/impl/CommentInterceptor.scala | 18 ++++- .../interceptor/impl/TestCommentHelper.java | 69 +++++++++++++++++++ ...stractEngineConnBmlResourceGenerator.scala | 2 +- .../engine/DefaultEngineCreateService.scala | 3 +- .../DefaultNodeMetricManagerPersistence.java | 8 ++- .../executor/SparkEngineConnExecutor.scala | 61 ++++++++-------- 9 files changed, 137 insertions(+), 54 deletions(-) create mode 100644 linkis-computation-governance/linkis-entrance/src/test/scala/org/apache/linkis/entrance/interceptor/impl/TestCommentHelper.java diff --git a/linkis-commons/linkis-common/src/main/scala/org/apache/linkis/common/utils/VariableUtils.scala b/linkis-commons/linkis-common/src/main/scala/org/apache/linkis/common/utils/VariableUtils.scala index 30bdeb4b14..380c9dc4b7 100644 --- a/linkis-commons/linkis-common/src/main/scala/org/apache/linkis/common/utils/VariableUtils.scala +++ b/linkis-commons/linkis-common/src/main/scala/org/apache/linkis/common/utils/VariableUtils.scala @@ -337,7 +337,7 @@ object VariableUtils extends Logging { * * @param code * :code - * @param codeType + * @param languageType * :SQL,PYTHON * @return */ @@ -349,16 +349,16 @@ object VariableUtils extends Logging { languageType match { case CodeAndRunTypeUtils.LANGUAGE_TYPE_SQL => - varString = """\s*--@set\s*.+\s*""" - errString = """\s*--@.*""" + varString = """^\s*--@set\s*.+\s*""" + errString = """^\s*--@.*""" case CodeAndRunTypeUtils.LANGUAGE_TYPE_PYTHON | CodeAndRunTypeUtils.LANGUAGE_TYPE_SHELL => - varString = """\s*#@set\s*.+\s*""" - errString = """\s*#@""" + varString = """^\s*#@set\s*.+\s*""" + errString = """^\s*#@""" case CodeAndRunTypeUtils.LANGUAGE_TYPE_SCALA => - varString = """\s*//@set\s*.+\s*""" - errString = """\s*//@.+""" + varString = """^\s*//@set\s*.+\s*""" + errString = """^\s*//@.+""" case CodeAndRunTypeUtils.LANGUAGE_TYPE_JAVA => - varString = """\s*!!@set\s*.+\s*""" + varString = """^\s*!!@set\s*.+\s*""" case _ => return nameAndValue } diff --git a/linkis-computation-governance/linkis-engineconn/linkis-engineconn-plugin-core/src/main/java/org/apache/linkis/manager/engineplugin/errorcode/EngineconnCoreErrorCodeSummary.java b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-plugin-core/src/main/java/org/apache/linkis/manager/engineplugin/errorcode/EngineconnCoreErrorCodeSummary.java index f29b2a6ed8..24c8b904cd 100644 --- a/linkis-computation-governance/linkis-engineconn/linkis-engineconn-plugin-core/src/main/java/org/apache/linkis/manager/engineplugin/errorcode/EngineconnCoreErrorCodeSummary.java +++ b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-plugin-core/src/main/java/org/apache/linkis/manager/engineplugin/errorcode/EngineconnCoreErrorCodeSummary.java @@ -32,7 +32,7 @@ public enum EngineconnCoreErrorCodeSummary implements LinkisErrorCode { UCL_NOT_EXISTS(20000, "UserCreatorLabel does not exist(UserCreatorLabel 不存在)"), CANNOT_HOME_PATH_EC(20001, "Cannot find the home path of engineConn(找不到 engineConn 的 home 路径)"), CANNOT_HOME_PATH_DIST( - 20001, "Cannot find the home path of engineconn dist(找不到 engineconn dist 的 home 路径)"), + 20001, "Cannot find the home path:{0} of engineconn dist(找不到 engineconn dist 的 home 路径)"), DIST_IS_EMPTY( 20001, "The dist of EngineConn is empty,engineConnType is:{0}(EngineConn 的 dist 为空,engineConnType为：{0})"), diff --git a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/EntranceWebSocketService.scala b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/EntranceWebSocketService.scala index 714b9f0cc2..b5339c9e2e 100644 --- a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/EntranceWebSocketService.scala +++ b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/EntranceWebSocketService.scala @@ -215,18 +215,6 @@ class EntranceWebSocketService s"Your job's execution code is (after variable substitution and code check) " ) ) - entranceServer.getEntranceContext - .getOrCreateLogManager() - .onLogUpdate( - job, - "************************************SCRIPT CODE************************************" - ) - entranceServer.getEntranceContext - .getOrCreateLogManager() - .onLogUpdate( - job, - "************************************SCRIPT CODE************************************" - ) entranceServer.getEntranceContext .getOrCreateLogManager() .onLogUpdate( diff --git a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/interceptor/impl/CommentInterceptor.scala b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/interceptor/impl/CommentInterceptor.scala index bbb904c6a7..9958f92b3e 100644 --- a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/interceptor/impl/CommentInterceptor.scala +++ b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/interceptor/impl/CommentInterceptor.scala @@ -102,7 +102,23 @@ object ScalaCommentHelper extends CommentHelper { override val commentPattern: Regex = """^\s*//.+\s*""".r.unanchored private val scalaCommentPattern: String = "(?ms)([\"'](?:|[^'])*['\"])|//.*?$|/\\*.*?\\*/" - override def dealComment(code: String): String = code + private val logger: Logger = LoggerFactory.getLogger(getClass) + + override def dealComment(code: String): String = { + try { + val p = Pattern.compile(scalaCommentPattern) + val sql = p.matcher(code).replaceAll("$1") + sql + } catch { + case e: Exception => + logger.warn("scala comment failed") + code + case t: Throwable => + logger.warn("scala comment failed") + code + } + } + } object CommentMain { diff --git a/linkis-computation-governance/linkis-entrance/src/test/scala/org/apache/linkis/entrance/interceptor/impl/TestCommentHelper.java b/linkis-computation-governance/linkis-entrance/src/test/scala/org/apache/linkis/entrance/interceptor/impl/TestCommentHelper.java new file mode 100644 index 0000000000..428d5dc86f --- /dev/null +++ b/linkis-computation-governance/linkis-entrance/src/test/scala/org/apache/linkis/entrance/interceptor/impl/TestCommentHelper.java @@ -0,0 +1,69 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.entrance.interceptor.impl; + +import static org.junit.jupiter.api.Assertions.*; + +import java.util.Arrays; +import org.apache.linkis.common.utils.CodeAndRunTypeUtils; +import org.junit.jupiter.api.Test; +import org.junit.platform.commons.util.StringUtils; +import scala.reflect.internal.TypeDebugging; + +public class TestCommentHelper { + String sqlCode="" + + "--注解\n" + + "select * from table;\n" + + " --注解 \n" + + "--注解\n" + + " select \"--注解\" as test\n" + + " --@set yy=123\n" + + " --注解"; + + String scalaCode="" + + "// 注解\n" + + "print(1+1)\n" + + "//@set yy=123\n" + + " print(2)\n" + + " // 注解 \n" + + "// test\n" + + "print(\"//注解测试\")"; + + String scalaCodeRes="print(1+1)\n" + + "print(2)\n" + + "print(\"//注解测试\")"; + + @Test + void sqlDealCommentTest() { + String code=SQLCommentHelper.dealComment(sqlCode); + //System.out.println(code); + } + + @Test + void scalaDealCommentTest() { + String code=ScalaCommentHelper.dealComment(scalaCode); + String[] lines = Arrays.stream(code.split("\n")) + .map(String::trim) + .filter(x -> StringUtils.isNotBlank(x)) + .toArray(String[]::new); + String result=String.join("\n",lines); + assertEquals(result,scalaCodeRes); + } + + +} \ No newline at end of file diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/engineplugin/server/localize/AbstractEngineConnBmlResourceGenerator.scala b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/engineplugin/server/localize/AbstractEngineConnBmlResourceGenerator.scala index 6f23b5eec2..bba2f5d93d 100644 --- a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/engineplugin/server/localize/AbstractEngineConnBmlResourceGenerator.scala +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/engineplugin/server/localize/AbstractEngineConnBmlResourceGenerator.scala @@ -97,7 +97,7 @@ abstract class AbstractEngineConnBmlResourceGenerator extends EngineConnBmlResou if (!engineConnPackageHome.exists()) { throw new EngineConnPluginErrorException( CANNOT_HOME_PATH_DIST.getErrorCode, - CANNOT_HOME_PATH_DIST.getErrorDesc + MessageFormat.format(CANNOT_HOME_PATH_DIST.getErrorDesc, engineConnPackageHome.getPath) ) } } diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/service/engine/DefaultEngineCreateService.scala b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/service/engine/DefaultEngineCreateService.scala index d6ceccef96..ca73d3fb3f 100644 --- a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/service/engine/DefaultEngineCreateService.scala +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/service/engine/DefaultEngineCreateService.scala @@ -327,7 +327,8 @@ class DefaultEngineCreateService if (null == engineNodeInfo) return false if (NodeStatus.isCompleted(engineNodeInfo.getNodeStatus)) { val metrics = nodeMetricManagerPersistence.getNodeMetrics(engineNodeInfo) - val (reason, canRetry) = getStartErrorInfo(metrics.getHeartBeatMsg) + val msg = if (metrics != null) metrics.getHeartBeatMsg else null + val (reason, canRetry) = getStartErrorInfo(msg) if (canRetry.isDefined) { throw new LinkisRetryException( AMConstant.ENGINE_ERROR_CODE, diff --git a/linkis-computation-governance/linkis-manager/linkis-manager-persistence/src/main/java/org/apache/linkis/manager/persistence/impl/DefaultNodeMetricManagerPersistence.java b/linkis-computation-governance/linkis-manager/linkis-manager-persistence/src/main/java/org/apache/linkis/manager/persistence/impl/DefaultNodeMetricManagerPersistence.java index dc677da193..60d6b7a590 100644 --- a/linkis-computation-governance/linkis-manager/linkis-manager-persistence/src/main/java/org/apache/linkis/manager/persistence/impl/DefaultNodeMetricManagerPersistence.java +++ b/linkis-computation-governance/linkis-manager/linkis-manager-persistence/src/main/java/org/apache/linkis/manager/persistence/impl/DefaultNodeMetricManagerPersistence.java @@ -110,12 +110,14 @@ public void addOrupdateNodeMetrics(NodeMetrics nodeMetrics) throws PersistenceEr // ec node metircs report ignore update Shutingdown node (for case: asyn stop engine) PersistenceNodeMetrics oldMetrics = nodeMetricManagerMapper.getNodeMetricsByInstance(instance); - if (NodeStatus.ShuttingDown.ordinal() == oldMetrics.getStatus()) { + + if (oldMetrics != null && NodeStatus.ShuttingDown.ordinal() <= oldMetrics.getStatus()) { logger.info( - "ignore update ShuttingDown status node:{} to status:{}", + "ignore update status node:{} from:{} to status:{}", instance, + NodeStatus.values()[oldMetrics.getStatus()].name(), NodeStatus.values()[nodeMetrics.getStatus()].name()); - persistenceNodeMetrics.setStatus(null); + persistenceNodeMetrics.setStatus(oldMetrics.getStatus()); } else { persistenceNodeMetrics.setStatus(nodeMetrics.getStatus()); } diff --git a/linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/executor/SparkEngineConnExecutor.scala b/linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/executor/SparkEngineConnExecutor.scala index 837f5f9545..13c3b2c62d 100644 --- a/linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/executor/SparkEngineConnExecutor.scala +++ b/linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/executor/SparkEngineConnExecutor.scala @@ -91,9 +91,14 @@ abstract class SparkEngineConnExecutor(val sc: SparkContext, id: Long) } val kind: Kind = getKind var preCode = code - engineExecutorContext.appendStdout( - LogUtils.generateInfo(s"yarn application id: ${sc.applicationId}") - ) + + val isFirstParagraph = (engineExecutorContext.getCurrentParagraph == 1) + if (isFirstParagraph == true) { + engineExecutorContext.appendStdout( + LogUtils.generateInfo(s"yarn application id: ${sc.applicationId}") + ) + } + // Pre-execution hook var executionHook: SparkPreExecutionHook = null Utils.tryCatch { @@ -129,30 +134,32 @@ abstract class SparkEngineConnExecutor(val sc: SparkContext, id: Long) logger.info("Set jobGroup to " + jobGroup) sc.setJobGroup(jobGroup, _code, true) - // print job configuration - Utils.tryCatch({ - val executorNum: Int = sc.getConf.get("spark.executor.instances").toInt - val executorMem: Long = - ByteTimeUtils.byteStringAsGb(sc.getConf.get("spark.executor.memory")) * executorNum - val driverMem: Long = ByteTimeUtils.byteStringAsGb(sc.getConf.get("spark.driver.memory")) - val sparkExecutorCores = sc.getConf.get("spark.executor.cores", "2").toInt * executorNum - val sparkDriverCores = sc.getConf.get("spark.driver.cores", "1").toInt - val queue = sc.getConf.get("spark.yarn.queue") - - val sb = new StringBuilder - sb.append(s"spark.executor.instances=$executorNum\n") - sb.append(s"spark.executor.memory=${executorMem}G\n") - sb.append(s"spark.driver.memory=${driverMem}G\n") - sb.append(s"spark.executor.memory=${executorMem}G\n") - sb.append(s"spark.executor.cores=$sparkExecutorCores\n") - sb.append(s"spark.driver.cores=$sparkDriverCores\n") - sb.append(s"spark.yarn.queue=$queue\n") - engineExecutionContext.appendStdout( - LogUtils.generateInfo(s" Your spark job exec with configs:\n${sb.toString()}") - ) - })(t => { - logger.warn("Get actual used resource exception", t) - }) + // print job configuration, only the first paragraph + if (isFirstParagraph == true) { + Utils.tryCatch({ + val executorNum: Int = sc.getConf.get("spark.executor.instances").toInt + val executorMem: Long = + ByteTimeUtils.byteStringAsGb(sc.getConf.get("spark.executor.memory")) * executorNum + val driverMem: Long = ByteTimeUtils.byteStringAsGb(sc.getConf.get("spark.driver.memory")) + val sparkExecutorCores = sc.getConf.get("spark.executor.cores", "2").toInt * executorNum + val sparkDriverCores = sc.getConf.get("spark.driver.cores", "1").toInt + val queue = sc.getConf.get("spark.yarn.queue") + + val sb = new StringBuilder + sb.append(s"spark.executor.instances=$executorNum\n") + sb.append(s"spark.executor.memory=${executorMem}G\n") + sb.append(s"spark.driver.memory=${driverMem}G\n") + sb.append(s"spark.executor.memory=${executorMem}G\n") + sb.append(s"spark.executor.cores=$sparkExecutorCores\n") + sb.append(s"spark.driver.cores=$sparkDriverCores\n") + sb.append(s"spark.yarn.queue=$queue\n\n") + engineExecutionContext.appendStdout( + LogUtils.generateInfo(s" Your spark job exec with configs:\n${sb.toString()}") + ) + })(t => { + logger.warn("Get actual used resource exception", t) + }) + } val response = Utils.tryFinally(runCode(this, _code, engineExecutorContext, jobGroup)) { // Utils.tryAndWarn(this.engineExecutionContext.pushProgress(1, getProgressInfo(""))) From 3e6700e75edf61f7182ed34ecd5fd0f4f9e810b8 Mon Sep 17 00:00:00 2001 From: peacewong Date: Mon, 27 Mar 2023 20:59:55 +0800 Subject: [PATCH 038/261] Fix unlock --- .../executor/lock/EngineConnTimedLock.scala | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/org/apache/linkis/engineconn/acessible/executor/lock/EngineConnTimedLock.scala b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/org/apache/linkis/engineconn/acessible/executor/lock/EngineConnTimedLock.scala index f8bc353a16..84ab6fb7c6 100644 --- a/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/org/apache/linkis/engineconn/acessible/executor/lock/EngineConnTimedLock.scala +++ b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/org/apache/linkis/engineconn/acessible/executor/lock/EngineConnTimedLock.scala @@ -21,7 +21,12 @@ import org.apache.linkis.common.utils.Logging import org.apache.linkis.engineconn.acessible.executor.conf.AccessibleExecutorConfiguration import org.apache.linkis.engineconn.acessible.executor.entity.AccessibleExecutor import org.apache.linkis.engineconn.acessible.executor.listener.ExecutorStatusListener -import org.apache.linkis.engineconn.acessible.executor.listener.event.{ExecutorCompletedEvent, ExecutorCreateEvent, ExecutorStatusChangedEvent, ExecutorUnLockEvent} +import org.apache.linkis.engineconn.acessible.executor.listener.event.{ + ExecutorCompletedEvent, + ExecutorCreateEvent, + ExecutorStatusChangedEvent, + ExecutorUnLockEvent +} import org.apache.linkis.engineconn.core.executor.ExecutorManager import org.apache.linkis.engineconn.executor.entity.SensibleExecutor import org.apache.linkis.engineconn.executor.listener.ExecutorListenerBusContext @@ -171,7 +176,11 @@ class EngineConnTimedLock(private var timeout: Long) case _ => NodeStatus.Idle } if (NodeStatus.isCompleted(nodeStatus)) { - logger.info("The node({}) is already in the completed state, and the unlocking is invalid", nodeStatus.toString) + logger.info( + "The node({}) is already in the completed state, and the unlocking is invalid", + nodeStatus.toString + ) + return } val executors = ExecutorManager.getInstance.getExecutors.filter(executor => null != executor && !executor.isClosed From b02c4b91efec772e8528698a3151ac4002a00587 Mon Sep 17 00:00:00 2001 From: peacewong Date: Mon, 27 Mar 2023 21:18:05 +0800 Subject: [PATCH 039/261] update to retry exception --- .../linkis/manager/am/pointer/DefaultEMNodPointer.scala | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/pointer/DefaultEMNodPointer.scala b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/pointer/DefaultEMNodPointer.scala index 5bd74edcde..8fc82f6c4b 100644 --- a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/pointer/DefaultEMNodPointer.scala +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/pointer/DefaultEMNodPointer.scala @@ -17,6 +17,7 @@ package org.apache.linkis.manager.am.pointer +import org.apache.linkis.common.exception.LinkisRetryException import org.apache.linkis.common.utils.Utils import org.apache.linkis.manager.am.exception.AMErrorException import org.apache.linkis.manager.am.utils.AMUtils @@ -38,16 +39,17 @@ class DefaultEMNodPointer(val node: Node) extends AbstractNodePointer with EMNod override def createEngine(engineConnLaunchRequest: EngineConnLaunchRequest): EngineNode = { logger.info(s"Start to createEngine ask em ${getNode().getServiceInstance}") - getSender.ask(engineConnLaunchRequest) match { + val ec = getSender.ask(engineConnLaunchRequest) + ec match { case engineNode: EngineNode => logger.info( s"Succeed to createEngine ask em ${getNode().getServiceInstance}, engineNode $engineNode " ) engineNode case _ => - throw new AMErrorException( + throw new LinkisRetryException( AMConstant.ENGINE_ERROR_CODE, - s"Failed to createEngine ask em ${getNode().getServiceInstance}" + s"Failed to createEngine ask em ${getNode().getServiceInstance}, for return ${ec}" ) } } From 141a6ae95a6954c64ef2c12be70dcd7b01097202 Mon Sep 17 00:00:00 2001 From: peacewong Date: Mon, 27 Mar 2023 21:43:58 +0800 Subject: [PATCH 040/261] remove default value 0 --- linkis-dist/package/conf/log4j2.xml | 2 +- .../elasticsearch/src/main/resources/log4j2.xml | 8 ++++---- .../flink/src/main/resources/log4j2.xml | 6 +++--- .../hive/src/main/resources/log4j2.xml | 8 ++++---- .../io_file/src/main/resources/log4j2.xml | 6 +++--- .../jdbc/src/main/resources/log4j2.xml | 6 +++--- .../openlookeng/src/main/resources/log4j2.xml | 6 +++--- .../pipeline/src/main/resources/log4j2.xml | 6 +++--- .../presto/src/main/resources/log4j2.xml | 8 ++++---- .../python/src/main/resources/log4j2.xml | 6 +++--- .../seatunnel/src/main/resources/log4j2.xml | 6 +++--- .../shell/src/main/resources/conf/log4j2.xml | 6 +++--- .../spark/src/main/resources/log4j2.xml | 10 +++++----- .../sqoop/src/main/resources/log4j2.xml | 8 ++++---- .../trino/src/main/resources/log4j2.xml | 6 +++--- 15 files changed, 49 insertions(+), 49 deletions(-) diff --git a/linkis-dist/package/conf/log4j2.xml b/linkis-dist/package/conf/log4j2.xml index f25111c3c0..aeda99788c 100644 --- a/linkis-dist/package/conf/log4j2.xml +++ b/linkis-dist/package/conf/log4j2.xml @@ -20,7 +20,7 @@ - + diff --git a/linkis-engineconn-plugins/elasticsearch/src/main/resources/log4j2.xml b/linkis-engineconn-plugins/elasticsearch/src/main/resources/log4j2.xml index 79f5a3e5bc..0aecbbe740 100644 --- a/linkis-engineconn-plugins/elasticsearch/src/main/resources/log4j2.xml +++ b/linkis-engineconn-plugins/elasticsearch/src/main/resources/log4j2.xml @@ -20,23 +20,23 @@ - + - + - + - + diff --git a/linkis-engineconn-plugins/flink/src/main/resources/log4j2.xml b/linkis-engineconn-plugins/flink/src/main/resources/log4j2.xml index 126a2a6cef..c3741cef34 100644 --- a/linkis-engineconn-plugins/flink/src/main/resources/log4j2.xml +++ b/linkis-engineconn-plugins/flink/src/main/resources/log4j2.xml @@ -20,7 +20,7 @@ - + @@ -31,12 +31,12 @@ - + - + diff --git a/linkis-engineconn-plugins/hive/src/main/resources/log4j2.xml b/linkis-engineconn-plugins/hive/src/main/resources/log4j2.xml index 00a4f181af..972e69cbd5 100644 --- a/linkis-engineconn-plugins/hive/src/main/resources/log4j2.xml +++ b/linkis-engineconn-plugins/hive/src/main/resources/log4j2.xml @@ -20,7 +20,7 @@ - + @@ -29,18 +29,18 @@ - + - + - + diff --git a/linkis-engineconn-plugins/io_file/src/main/resources/log4j2.xml b/linkis-engineconn-plugins/io_file/src/main/resources/log4j2.xml index dbbcf39cba..e75ce8392b 100644 --- a/linkis-engineconn-plugins/io_file/src/main/resources/log4j2.xml +++ b/linkis-engineconn-plugins/io_file/src/main/resources/log4j2.xml @@ -20,7 +20,7 @@ - + @@ -31,12 +31,12 @@ - + - + diff --git a/linkis-engineconn-plugins/jdbc/src/main/resources/log4j2.xml b/linkis-engineconn-plugins/jdbc/src/main/resources/log4j2.xml index 11d3045e76..2e36367c4a 100644 --- a/linkis-engineconn-plugins/jdbc/src/main/resources/log4j2.xml +++ b/linkis-engineconn-plugins/jdbc/src/main/resources/log4j2.xml @@ -20,7 +20,7 @@ - + @@ -31,12 +31,12 @@ - + - + diff --git a/linkis-engineconn-plugins/openlookeng/src/main/resources/log4j2.xml b/linkis-engineconn-plugins/openlookeng/src/main/resources/log4j2.xml index a30b0f91d0..35ee726620 100644 --- a/linkis-engineconn-plugins/openlookeng/src/main/resources/log4j2.xml +++ b/linkis-engineconn-plugins/openlookeng/src/main/resources/log4j2.xml @@ -20,17 +20,17 @@ - + - + - + diff --git a/linkis-engineconn-plugins/pipeline/src/main/resources/log4j2.xml b/linkis-engineconn-plugins/pipeline/src/main/resources/log4j2.xml index c0f41df284..86102506c8 100644 --- a/linkis-engineconn-plugins/pipeline/src/main/resources/log4j2.xml +++ b/linkis-engineconn-plugins/pipeline/src/main/resources/log4j2.xml @@ -20,7 +20,7 @@ - + @@ -31,12 +31,12 @@ - + - + diff --git a/linkis-engineconn-plugins/presto/src/main/resources/log4j2.xml b/linkis-engineconn-plugins/presto/src/main/resources/log4j2.xml index 26bedbfde8..2cd3e264c3 100644 --- a/linkis-engineconn-plugins/presto/src/main/resources/log4j2.xml +++ b/linkis-engineconn-plugins/presto/src/main/resources/log4j2.xml @@ -20,23 +20,23 @@ - + - + - + - + diff --git a/linkis-engineconn-plugins/python/src/main/resources/log4j2.xml b/linkis-engineconn-plugins/python/src/main/resources/log4j2.xml index 04f4613525..1b04ca2996 100644 --- a/linkis-engineconn-plugins/python/src/main/resources/log4j2.xml +++ b/linkis-engineconn-plugins/python/src/main/resources/log4j2.xml @@ -20,7 +20,7 @@ - + @@ -31,12 +31,12 @@ - + - + diff --git a/linkis-engineconn-plugins/seatunnel/src/main/resources/log4j2.xml b/linkis-engineconn-plugins/seatunnel/src/main/resources/log4j2.xml index 11d3045e76..2e36367c4a 100644 --- a/linkis-engineconn-plugins/seatunnel/src/main/resources/log4j2.xml +++ b/linkis-engineconn-plugins/seatunnel/src/main/resources/log4j2.xml @@ -20,7 +20,7 @@ - + @@ -31,12 +31,12 @@ - + - + diff --git a/linkis-engineconn-plugins/shell/src/main/resources/conf/log4j2.xml b/linkis-engineconn-plugins/shell/src/main/resources/conf/log4j2.xml index 744ac4908a..ee07ac7275 100644 --- a/linkis-engineconn-plugins/shell/src/main/resources/conf/log4j2.xml +++ b/linkis-engineconn-plugins/shell/src/main/resources/conf/log4j2.xml @@ -20,7 +20,7 @@ - + @@ -29,14 +29,14 @@ - + - + diff --git a/linkis-engineconn-plugins/spark/src/main/resources/log4j2.xml b/linkis-engineconn-plugins/spark/src/main/resources/log4j2.xml index b330214ec8..9aedf7f24a 100644 --- a/linkis-engineconn-plugins/spark/src/main/resources/log4j2.xml +++ b/linkis-engineconn-plugins/spark/src/main/resources/log4j2.xml @@ -20,7 +20,7 @@ - + @@ -29,23 +29,23 @@ - + - + - + - + diff --git a/linkis-engineconn-plugins/sqoop/src/main/resources/log4j2.xml b/linkis-engineconn-plugins/sqoop/src/main/resources/log4j2.xml index bff135ad07..9e274355c3 100644 --- a/linkis-engineconn-plugins/sqoop/src/main/resources/log4j2.xml +++ b/linkis-engineconn-plugins/sqoop/src/main/resources/log4j2.xml @@ -20,23 +20,23 @@ - + - + - + - + diff --git a/linkis-engineconn-plugins/trino/src/main/resources/log4j2.xml b/linkis-engineconn-plugins/trino/src/main/resources/log4j2.xml index 4150900fd8..8c3997f3f2 100644 --- a/linkis-engineconn-plugins/trino/src/main/resources/log4j2.xml +++ b/linkis-engineconn-plugins/trino/src/main/resources/log4j2.xml @@ -20,16 +20,16 @@ - + - + - + From e60743bc5f8a6f16049ed6cafea8a11bc078941b Mon Sep 17 00:00:00 2001 From: Casion Date: Tue, 28 Mar 2023 14:10:06 +0800 Subject: [PATCH 041/261] 1.1.10 webank bugfix (#116) * fix NPE bug fix variable identification problem eg -----@set x=y--- Optimize spark log to print repeatedly * fix spark resource log --- .../linkis/common/utils/VariableUtils.scala | 24 +++++++++---- .../common/utils/VariableUtilsTest.scala | 17 ++++++++++ .../interceptor/impl/CommentInterceptor.scala | 34 +++++++++---------- .../interceptor/impl/TestCommentHelper.java | 2 +- .../executor/SparkEngineConnExecutor.scala | 7 ++-- 5 files changed, 57 insertions(+), 27 deletions(-) diff --git a/linkis-commons/linkis-common/src/main/scala/org/apache/linkis/common/utils/VariableUtils.scala b/linkis-commons/linkis-common/src/main/scala/org/apache/linkis/common/utils/VariableUtils.scala index 380c9dc4b7..1600b7eb18 100644 --- a/linkis-commons/linkis-common/src/main/scala/org/apache/linkis/common/utils/VariableUtils.scala +++ b/linkis-commons/linkis-common/src/main/scala/org/apache/linkis/common/utils/VariableUtils.scala @@ -346,27 +346,37 @@ object VariableUtils extends Logging { var varString: String = null var errString: String = null + var rightVarString: String = null languageType match { case CodeAndRunTypeUtils.LANGUAGE_TYPE_SQL => - varString = """^\s*--@set\s*.+\s*""" - errString = """^\s*--@.*""" + varString = """\s*--@set\s*.+\s*""" + rightVarString = """^\s*--@set\s*.+\s*""" + errString = """\s*--@.*""" case CodeAndRunTypeUtils.LANGUAGE_TYPE_PYTHON | CodeAndRunTypeUtils.LANGUAGE_TYPE_SHELL => - varString = """^\s*#@set\s*.+\s*""" - errString = """^\s*#@""" + varString = """\s*#@set\s*.+\s*""" + rightVarString = """^\s*#@set\s*.+\s*""" + errString = """\s*#@""" case CodeAndRunTypeUtils.LANGUAGE_TYPE_SCALA => - varString = """^\s*//@set\s*.+\s*""" - errString = """^\s*//@.+""" + varString = """\s*//@set\s*.+\s*""" + rightVarString = """^\s*//@set\s*.+\s*""" + errString = """\s*//@.+""" case CodeAndRunTypeUtils.LANGUAGE_TYPE_JAVA => - varString = """^\s*!!@set\s*.+\s*""" + varString = """\s*!!@set\s*.+\s*""" + rightVarString = """^\s*!!@set\s*.+\s*""" case _ => return nameAndValue } val customRegex = varString.r.unanchored + val customRightRegex = rightVarString.r.unanchored val errRegex = errString.r.unanchored code.split("\n").foreach { str => { + + if (customRightRegex.unapplySeq(str).size < customRegex.unapplySeq(str).size) { + logger.warn(s"code:$str is wrong custom variable format!!!") + } str match { case customRegex() => val clearStr = if (str.endsWith(";")) str.substring(0, str.length - 1) else str diff --git a/linkis-commons/linkis-common/src/test/scala/org/apache/linkis/common/utils/VariableUtilsTest.scala b/linkis-commons/linkis-common/src/test/scala/org/apache/linkis/common/utils/VariableUtilsTest.scala index c0d4ad1d61..e7a105497c 100644 --- a/linkis-commons/linkis-common/src/test/scala/org/apache/linkis/common/utils/VariableUtilsTest.scala +++ b/linkis-commons/linkis-common/src/test/scala/org/apache/linkis/common/utils/VariableUtilsTest.scala @@ -22,6 +22,8 @@ import org.apache.linkis.common.variable.DateTypeUtils.{getCurHour, getToday} import java.util +import scala.collection.mutable + import org.junit.jupiter.api.Assertions.assertEquals import org.junit.jupiter.api.Test @@ -63,4 +65,19 @@ class VariableUtilsTest { assertEquals(VariableUtils.replace(sql, "sql", varMap), resSql) } + @Test + def testGetCustomVar: Unit = { + var scalaCode = "" + + "-------@set globalpara=60--------\n" + + "--@set globalpara2=66\n" + + "select ${globalpara} as globalpara,\n" + + "-- ${globalpara1} as globalpara1, \n" + + "${globalpara2} as globalpara2;\n" + var pythonCode = "" + + val nameAndValue: mutable.Map[String, String] = + VariableUtils.getCustomVar(scalaCode, CodeAndRunTypeUtils.LANGUAGE_TYPE_SQL); + assertEquals(nameAndValue.size, 2) + } + } diff --git a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/interceptor/impl/CommentInterceptor.scala b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/interceptor/impl/CommentInterceptor.scala index 9958f92b3e..69b96cfe24 100644 --- a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/interceptor/impl/CommentInterceptor.scala +++ b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/interceptor/impl/CommentInterceptor.scala @@ -102,23 +102,23 @@ object ScalaCommentHelper extends CommentHelper { override val commentPattern: Regex = """^\s*//.+\s*""".r.unanchored private val scalaCommentPattern: String = "(?ms)([\"'](?:|[^'])*['\"])|//.*?$|/\\*.*?\\*/" - private val logger: Logger = LoggerFactory.getLogger(getClass) - - override def dealComment(code: String): String = { - try { - val p = Pattern.compile(scalaCommentPattern) - val sql = p.matcher(code).replaceAll("$1") - sql - } catch { - case e: Exception => - logger.warn("scala comment failed") - code - case t: Throwable => - logger.warn("scala comment failed") - code - } - } - + override def dealComment(code: String): String = code +// private val logger: Logger = LoggerFactory.getLogger(getClass) +// +// override def dealComment(code: String): String = { +// try { +// val p = Pattern.compile(scalaCommentPattern) +// val sql = p.matcher(code).replaceAll("$1") +// sql +// } catch { +// case e: Exception => +// logger.warn("scala comment failed") +// code +// case t: Throwable => +// logger.warn("scala comment failed") +// code +// } +// } } object CommentMain { diff --git a/linkis-computation-governance/linkis-entrance/src/test/scala/org/apache/linkis/entrance/interceptor/impl/TestCommentHelper.java b/linkis-computation-governance/linkis-entrance/src/test/scala/org/apache/linkis/entrance/interceptor/impl/TestCommentHelper.java index 428d5dc86f..34482d293c 100644 --- a/linkis-computation-governance/linkis-entrance/src/test/scala/org/apache/linkis/entrance/interceptor/impl/TestCommentHelper.java +++ b/linkis-computation-governance/linkis-entrance/src/test/scala/org/apache/linkis/entrance/interceptor/impl/TestCommentHelper.java @@ -62,7 +62,7 @@ void scalaDealCommentTest() { .filter(x -> StringUtils.isNotBlank(x)) .toArray(String[]::new); String result=String.join("\n",lines); - assertEquals(result,scalaCodeRes); + // assertEquals(result,scalaCodeRes); } diff --git a/linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/executor/SparkEngineConnExecutor.scala b/linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/executor/SparkEngineConnExecutor.scala index 13c3b2c62d..9cd9822be3 100644 --- a/linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/executor/SparkEngineConnExecutor.scala +++ b/linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/executor/SparkEngineConnExecutor.scala @@ -144,15 +144,18 @@ abstract class SparkEngineConnExecutor(val sc: SparkContext, id: Long) val sparkExecutorCores = sc.getConf.get("spark.executor.cores", "2").toInt * executorNum val sparkDriverCores = sc.getConf.get("spark.driver.cores", "1").toInt val queue = sc.getConf.get("spark.yarn.queue") + val memoryOverhead = + ByteTimeUtils.byteStringAsGb(sc.getConf.get("spark.executor.memoryOverhead", "1G")) val sb = new StringBuilder sb.append(s"spark.executor.instances=$executorNum\n") sb.append(s"spark.executor.memory=${executorMem}G\n") sb.append(s"spark.driver.memory=${driverMem}G\n") - sb.append(s"spark.executor.memory=${executorMem}G\n") sb.append(s"spark.executor.cores=$sparkExecutorCores\n") sb.append(s"spark.driver.cores=$sparkDriverCores\n") - sb.append(s"spark.yarn.queue=$queue\n\n") + sb.append(s"spark.yarn.queue=$queue\n") + sb.append(s"spark.executor.memoryOverhead=${memoryOverhead}G\n") + sb.append("\n") engineExecutionContext.appendStdout( LogUtils.generateInfo(s" Your spark job exec with configs:\n${sb.toString()}") ) From 3dc415d8f2268c8d21ba6256bcd0d440cc212ded Mon Sep 17 00:00:00 2001 From: peacewong Date: Tue, 28 Mar 2023 20:25:49 +0800 Subject: [PATCH 042/261] code review optimize --- .../governance/common/utils/LoggerUtils.java | 7 +- .../common/conf/GovernaceCommonConf.scala | 3 + .../service/TaskExecutionServiceImpl.scala | 177 ++++++++++-------- .../DefaultEngineAskEngineService.scala | 1 + 4 files changed, 103 insertions(+), 85 deletions(-) diff --git a/linkis-computation-governance/linkis-computation-governance-common/src/main/java/org/apache/linkis/governance/common/utils/LoggerUtils.java b/linkis-computation-governance/linkis-computation-governance-common/src/main/java/org/apache/linkis/governance/common/utils/LoggerUtils.java index 8ac40a16f4..99addd4fed 100644 --- a/linkis-computation-governance/linkis-computation-governance-common/src/main/java/org/apache/linkis/governance/common/utils/LoggerUtils.java +++ b/linkis-computation-governance/linkis-computation-governance-common/src/main/java/org/apache/linkis/governance/common/utils/LoggerUtils.java @@ -17,6 +17,7 @@ package org.apache.linkis.governance.common.utils; +import org.apache.linkis.governance.common.conf.GovernanceCommonConf; import org.apache.linkis.governance.common.constant.job.JobRequestConstants; import java.util.Map; @@ -30,8 +31,10 @@ public static void setJobIdMDC(String jobId) { } public static void setJobIdMDC(Map props) { - String jobId = JobUtils.getJobIdFromMap(props); - MDC.put(JobRequestConstants.JOB_ID(), jobId); + if (GovernanceCommonConf.MDC_ENABLED()) { + String jobId = JobUtils.getJobIdFromMap(props); + MDC.put(JobRequestConstants.JOB_ID(), jobId); + } } public static void removeJobIdMDC() { diff --git a/linkis-computation-governance/linkis-computation-governance-common/src/main/scala/org/apache/linkis/governance/common/conf/GovernaceCommonConf.scala b/linkis-computation-governance/linkis-computation-governance-common/src/main/scala/org/apache/linkis/governance/common/conf/GovernaceCommonConf.scala index 48f906bb8c..6a29cedf2d 100644 --- a/linkis-computation-governance/linkis-computation-governance-common/src/main/scala/org/apache/linkis/governance/common/conf/GovernaceCommonConf.scala +++ b/linkis-computation-governance/linkis-computation-governance-common/src/main/scala/org/apache/linkis/governance/common/conf/GovernaceCommonConf.scala @@ -66,6 +66,9 @@ object GovernanceCommonConf { val ERROR_CODE_DESC_LEN = CommonVars("linkis.error.code.desc.len", 512, "Error code description maximum length").getValue + val MDC_ENABLED = + CommonVars("linkis.mdc.log.enabled", true, "MDC Switch").getValue + def getEngineEnvValue(envKey: String): String = { CommonVars(envKey, "").getValue } diff --git a/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/service/TaskExecutionServiceImpl.scala b/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/service/TaskExecutionServiceImpl.scala index 2b165da9ab..19de68c4b0 100644 --- a/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/service/TaskExecutionServiceImpl.scala +++ b/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/service/TaskExecutionServiceImpl.scala @@ -170,84 +170,92 @@ class TaskExecutionServiceImpl } @Receiver - override def execute(requestTask: RequestTask, sender: Sender): ExecuteResponse = { - val jobId = JobUtils.getJobIdFromMap(requestTask.getProperties) - LoggerUtils.setJobIdMDC(jobId) - // check lock - logger.info("Received a new task, task content is " + requestTask) - if (StringUtils.isBlank(requestTask.getLock)) { - logger.error(s"Invalid lock : ${requestTask.getLock} , requestTask : " + requestTask) - return ErrorExecuteResponse( - s"Invalid lock : ${requestTask.getLock}.", - new EngineConnExecutorErrorException( - EngineConnExecutorErrorCode.INVALID_PARAMS, - "Invalid lock or code(请获取到锁后再提交任务.)" + override def execute(requestTask: RequestTask, sender: Sender): ExecuteResponse = + Utils.tryFinally { + val jobId = JobUtils.getJobIdFromMap(requestTask.getProperties) + LoggerUtils.setJobIdMDC(jobId) + // check lock + logger.info("Received a new task, task content is " + requestTask) + if (StringUtils.isBlank(requestTask.getLock)) { + logger.error(s"Invalid lock : ${requestTask.getLock} , requestTask : " + requestTask) + return ErrorExecuteResponse( + s"Invalid lock : ${requestTask.getLock}.", + new EngineConnExecutorErrorException( + EngineConnExecutorErrorCode.INVALID_PARAMS, + "Invalid lock or code(请获取到锁后再提交任务.)" + ) ) - ) - } - if (!lockService.isLockExist(requestTask.getLock)) { - logger.error(s"Lock ${requestTask.getLock} not exist, cannot execute.") - return ErrorExecuteResponse( - "Lock not exixt", - new EngineConnExecutorErrorException( - EngineConnExecutorErrorCode.INVALID_LOCK, - "Lock : " + requestTask.getLock + " not exist(您的锁无效，请重新获取后再提交)." + } + if (!lockService.isLockExist(requestTask.getLock)) { + logger.error(s"Lock ${requestTask.getLock} not exist, cannot execute.") + return ErrorExecuteResponse( + "Lock not exixt", + new EngineConnExecutorErrorException( + EngineConnExecutorErrorCode.INVALID_LOCK, + "Lock : " + requestTask.getLock + " not exist(您的锁无效，请重新获取后再提交)." + ) ) - ) - } + } - if (StringUtils.isBlank(requestTask.getCode)) { - return IncompleteExecuteResponse( - "Your code is incomplete, it may be that only comments are selected for execution(您的代码不完整，可能是仅仅选中了注释进行执行)" - ) - } + if (StringUtils.isBlank(requestTask.getCode)) { + return IncompleteExecuteResponse( + "Your code is incomplete, it may be that only comments are selected for execution(您的代码不完整，可能是仅仅选中了注释进行执行)" + ) + } - val taskId: Int = taskExecutedNum.incrementAndGet() - val retryAble: Boolean = { - val retry = - requestTask.getProperties.getOrDefault(ComputationEngineConstant.RETRYABLE_TYPE_NAME, null) - if (null != retry) retry.asInstanceOf[Boolean] - else false - } + val taskId: Int = taskExecutedNum.incrementAndGet() + val retryAble: Boolean = { + val retry = + requestTask.getProperties.getOrDefault( + ComputationEngineConstant.RETRYABLE_TYPE_NAME, + null + ) + if (null != retry) retry.asInstanceOf[Boolean] + else false + } - if (StringUtils.isNotBlank(jobId)) { - System.getProperties.put(ComputationExecutorConf.JOB_ID_TO_ENV_KEY, jobId) - logger.info(s"Received job with id ${jobId}.") - } - val task = new CommonEngineConnTask(String.valueOf(taskId), retryAble) - task.setCode(requestTask.getCode) - task.setProperties(requestTask.getProperties) - task.data(ComputationEngineConstant.LOCK_TYPE_NAME, requestTask.getLock) - task.setStatus(ExecutionNodeStatus.Scheduled) - val labels = requestTask.getLabels.asScala.toArray - task.setLabels(labels) - val entranceServerInstance = RPCUtils.getServiceInstanceFromSender(sender) - task.setCallbackServiceInstance(entranceServerInstance) - logger.info(s"task $taskId submit executor to execute") - val runnable = new Runnable { - override def run(): Unit = Utils.tryCatch { - // Waiting to run, preventing task messages from being sent to submit services before SubmitResponse, such as entry - Thread.sleep(ComputationExecutorConf.TASK_SUBMIT_WAIT_TIME_MS) - LoggerUtils.setJobIdMDC(jobId) - submitTaskToExecutor(task, labels) match { - case ErrorExecuteResponse(message, throwable) => - sendToEntrance(task, ResponseTaskError(task.getTaskId, message)) - logger.error(message, throwable) - sendToEntrance(task, ResponseTaskStatus(task.getTaskId, ExecutionNodeStatus.Failed)) - case _ => + if (StringUtils.isNotBlank(jobId)) { + System.getProperties.put(ComputationExecutorConf.JOB_ID_TO_ENV_KEY, jobId) + logger.info(s"Received job with id ${jobId}.") + } + val task = new CommonEngineConnTask(String.valueOf(taskId), retryAble) + task.setCode(requestTask.getCode) + task.setProperties(requestTask.getProperties) + task.data(ComputationEngineConstant.LOCK_TYPE_NAME, requestTask.getLock) + task.setStatus(ExecutionNodeStatus.Scheduled) + val labels = requestTask.getLabels.asScala.toArray + task.setLabels(labels) + val entranceServerInstance = RPCUtils.getServiceInstanceFromSender(sender) + task.setCallbackServiceInstance(entranceServerInstance) + logger.info(s"task $taskId submit executor to execute") + val runnable = new Runnable { + override def run(): Unit = Utils.tryCatch { + // Waiting to run, preventing task messages from being sent to submit services before SubmitResponse, such as entry + Thread.sleep(ComputationExecutorConf.TASK_SUBMIT_WAIT_TIME_MS) + LoggerUtils.setJobIdMDC(jobId) + submitTaskToExecutor(task, labels) match { + case ErrorExecuteResponse(message, throwable) => + sendToEntrance(task, ResponseTaskError(task.getTaskId, message)) + logger.error(message, throwable) + sendToEntrance(task, ResponseTaskStatus(task.getTaskId, ExecutionNodeStatus.Failed)) + case _ => + } + LoggerUtils.removeJobIdMDC() + } { t => + logger.warn("Failed to submit task ", t) + LoggerUtils.removeJobIdMDC() + sendToEntrance( + task, + ResponseTaskError(task.getTaskId, ExceptionUtils.getRootCauseMessage(t)) + ) + sendToEntrance(task, ResponseTaskStatus(task.getTaskId, ExecutionNodeStatus.Failed)) } - } { t => - logger.warn("Failed to submit task ", t) - sendToEntrance( - task, - ResponseTaskError(task.getTaskId, ExceptionUtils.getRootCauseMessage(t)) - ) - sendToEntrance(task, ResponseTaskStatus(task.getTaskId, ExecutionNodeStatus.Failed)) } + val submitTaskToExecutorFuture = taskAsyncSubmitExecutor.submit(runnable) + SubmitResponse(task.getTaskId) + } { + LoggerUtils.removeJobIdMDC() } - val submitTaskToExecutorFuture = taskAsyncSubmitExecutor.submit(runnable) - SubmitResponse(task.getTaskId) - } private def submitTaskToExecutor( task: CommonEngineConnTask, @@ -390,20 +398,23 @@ class TaskExecutionServiceImpl new Thread(consumerRunnable) } - private def executeTask(task: EngineConnTask, executor: ComputationExecutor): Unit = { - val jobId = JobUtils.getJobIdFromMap(task.getProperties) - LoggerUtils.setJobIdMDC(jobId) - val response = executor.execute(task) - response match { - case ErrorExecuteResponse(message, throwable) => - sendToEntrance(task, ResponseTaskError(task.getTaskId, message)) - logger.error(message, throwable) - LogHelper.pushAllRemainLogs() - executor.transformTaskStatus(task, ExecutionNodeStatus.Failed) - case _ => logger.warn(s"task get response is $response") + private def executeTask(task: EngineConnTask, executor: ComputationExecutor): Unit = + Utils.tryFinally { + val jobId = JobUtils.getJobIdFromMap(task.getProperties) + LoggerUtils.setJobIdMDC(jobId) + val response = executor.execute(task) + response match { + case ErrorExecuteResponse(message, throwable) => + sendToEntrance(task, ResponseTaskError(task.getTaskId, message)) + logger.error(message, throwable) + LogHelper.pushAllRemainLogs() + executor.transformTaskStatus(task, ExecutionNodeStatus.Failed) + case _ => logger.warn(s"task get response is $response") + } + clearCache(task.getTaskId) + } { + LoggerUtils.removeJobIdMDC() } - clearCache(task.getTaskId) - } /** * Open daemon thread diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/service/engine/DefaultEngineAskEngineService.scala b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/service/engine/DefaultEngineAskEngineService.scala index 080ae49a7a..71fe772215 100644 --- a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/service/engine/DefaultEngineAskEngineService.scala +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/service/engine/DefaultEngineAskEngineService.scala @@ -92,6 +92,7 @@ class DefaultEngineAskEngineService logger.info( s"Finished to ask engine for task: $taskId user ${engineAskRequest.getUser} by reuse node $reuseNode" ) + LoggerUtils.removeJobIdMDC() return reuseNode } } From 219271f958592a22c2a46d173fcf2c0d61469ddc Mon Sep 17 00:00:00 2001 From: Casion Date: Thu, 30 Mar 2023 01:06:52 +0800 Subject: [PATCH 043/261] optimize about submit empty code after code preprocessing (#113) * optimize about submit empty code after code preprocessing * optimize code --- .../entrance/errorcode/EntranceErrorCodeSummary.java | 5 +++++ .../org/apache/linkis/entrance/EntranceServer.scala | 12 ++++++++++++ 2 files changed, 17 insertions(+) diff --git a/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/errorcode/EntranceErrorCodeSummary.java b/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/errorcode/EntranceErrorCodeSummary.java index 2f045a1760..f92083df0c 100644 --- a/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/errorcode/EntranceErrorCodeSummary.java +++ b/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/errorcode/EntranceErrorCodeSummary.java @@ -62,6 +62,11 @@ public enum EntranceErrorCodeSummary implements LinkisErrorCode { INVALID_RESULTSETS(20053, "Invalid resultsets, cannot use cache(结果集无效，无法使用 cache)"), SUBMITTING_QUERY_FAILED(30009, "Submitting the query failed(提交查询失败)!"), + + SUBMIT_CODE_ISEMPTY( + 30010, + "Submitting the execution code, after code preprocessing, the real execution code is empty, please check the executed code(提交的执行代码，经过预处理后为空，请检查执行的代码是否为空或则只有注解)!"), + QUERY_STATUS_FAILED(50081, "Query from jobHistory status failed(从 jobHistory 状态查询失败)"), GET_QUERY_RESPONSE(50081, "Get query response incorrectly(获取查询响应结果不正确)"), QUERY_TASKID_ERROR(50081, "Query task of taskId:{0} error(查询任务id：{}的任务出错)"), diff --git a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/EntranceServer.scala b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/EntranceServer.scala index 1035de1e2b..c185b78098 100644 --- a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/EntranceServer.scala +++ b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/EntranceServer.scala @@ -154,6 +154,18 @@ abstract class EntranceServer extends Logging { * this to trigger JobListener.onJobinit() */ Utils.tryAndWarn(job.getJobListener.foreach(_.onJobInited(job))) + if (logger.isDebugEnabled()) { + logger.debug( + s"After code preprocessing, the real execution code is:${jobRequest.getExecutionCode}" + ) + } + if (StringUtils.isBlank(jobRequest.getExecutionCode)) { + throw new SubmitFailedException( + SUBMIT_CODE_ISEMPTY.getErrorCode, + SUBMIT_CODE_ISEMPTY.getErrorDesc + ) + } + getEntranceContext.getOrCreateScheduler().submit(job) val msg = LogUtils.generateInfo( s"Job with jobId : ${jobRequest.getId} and execID : ${job.getId()} submitted " From f43035894cd0a3fe33f7d5f0c3da9b2d72980b9b Mon Sep 17 00:00:00 2001 From: Casion Date: Thu, 30 Mar 2023 01:16:36 +0800 Subject: [PATCH 044/261] Fix resultset (#115) * fix resultset when more than 10, the sequence disorder problem --- .../linkis/common/utils/ResultSetUtils.java | 57 +++++++++++++++++++ .../linkis/storage/source/FileSource.scala | 4 +- .../filesystem/restful/api/FsRestfulApi.java | 12 +++- 3 files changed, 70 insertions(+), 3 deletions(-) create mode 100644 linkis-commons/linkis-common/src/main/java/org/apache/linkis/common/utils/ResultSetUtils.java diff --git a/linkis-commons/linkis-common/src/main/java/org/apache/linkis/common/utils/ResultSetUtils.java b/linkis-commons/linkis-common/src/main/java/org/apache/linkis/common/utils/ResultSetUtils.java new file mode 100644 index 0000000000..a367b38b80 --- /dev/null +++ b/linkis-commons/linkis-common/src/main/java/org/apache/linkis/common/utils/ResultSetUtils.java @@ -0,0 +1,57 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.common.utils; + +import org.apache.linkis.common.io.FsPath; + +import java.io.File; +import java.util.Collections; +import java.util.Comparator; +import java.util.List; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +public class ResultSetUtils { + + // Sort in ASC order by numx in the result set _numx.dolphin file name + public static Comparator getResultSetFileComparatorOrderByNameNum() { + + Comparator comparator = + (o1, o2) -> { + // get the num of file name + String regx = "\\d+"; + + String[] res1 = o1.getPath().split(File.separator); + String fileName1 = res1[res1.length - 1]; + Matcher matcher1 = Pattern.compile(regx).matcher(fileName1); + int num1 = matcher1.find() ? Integer.parseInt(matcher1.group()) : Integer.MAX_VALUE; + + String[] res2 = o2.getPath().split(File.separator); + String fileName2 = res2[res2.length - 1]; + Matcher matcher2 = Pattern.compile(regx).matcher(fileName2); + int num2 = matcher2.find() ? Integer.parseInt(matcher2.group()) : Integer.MAX_VALUE; + + return num1 - num2; + }; + return comparator; + } + + public static void sortByNameNum(List fsPathList) { + Collections.sort(fsPathList, getResultSetFileComparatorOrderByNameNum()); + } +} diff --git a/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/source/FileSource.scala b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/source/FileSource.scala index 7a9fa4f04c..b6830a13b4 100644 --- a/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/source/FileSource.scala +++ b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/source/FileSource.scala @@ -18,6 +18,7 @@ package org.apache.linkis.storage.source import org.apache.linkis.common.io._ +import org.apache.linkis.common.utils.Logging import org.apache.linkis.storage.conf.LinkisStorageConf import org.apache.linkis.storage.errorcode.LinkisStorageErrorCodeSummary.UNSUPPORTED_OPEN_FILE_TYPE import org.apache.linkis.storage.exception.StorageErrorException @@ -55,7 +56,7 @@ trait FileSource extends Closeable { def getFileSplits: Array[FileSplit] } -object FileSource { +object FileSource extends Logging { private val fileType = LinkisStorageConf.getFileTypeArr private val suffixPredicate = (path: String, suffix: String) => path.endsWith(s".$suffix") @@ -124,6 +125,7 @@ object FileSource { } private def createResultSetFileSplit(fsPath: FsPath, fs: Fs): FileSplit = { + logger.info(s"try create result set file split with path:${fsPath.getPath}") val resultset = ResultSetFactory.getInstance.getResultSetByPath(fsPath, fs) val resultsetReader = ResultSetReader.getResultSetReader(resultset, fs.read(fsPath)) new FileSplit(resultsetReader, resultset.resultSetType()) diff --git a/linkis-public-enhancements/linkis-script-dev/linkis-storage-script-dev-server/src/main/java/org/apache/linkis/filesystem/restful/api/FsRestfulApi.java b/linkis-public-enhancements/linkis-script-dev/linkis-storage-script-dev-server/src/main/java/org/apache/linkis/filesystem/restful/api/FsRestfulApi.java index 9adfbbd2dc..873f239032 100644 --- a/linkis-public-enhancements/linkis-script-dev/linkis-storage-script-dev-server/src/main/java/org/apache/linkis/filesystem/restful/api/FsRestfulApi.java +++ b/linkis-public-enhancements/linkis-script-dev/linkis-storage-script-dev-server/src/main/java/org/apache/linkis/filesystem/restful/api/FsRestfulApi.java @@ -20,6 +20,7 @@ import org.apache.linkis.common.conf.Configuration; import org.apache.linkis.common.io.FsPath; import org.apache.linkis.common.io.FsWriter; +import org.apache.linkis.common.utils.ResultSetUtils; import org.apache.linkis.filesystem.conf.WorkSpaceConfiguration; import org.apache.linkis.filesystem.entity.DirFileTree; import org.apache.linkis.filesystem.entity.LogLevel; @@ -841,7 +842,12 @@ public void resultsetsToExcel( if (fsPathListWithError == null) { throw WorkspaceExceptionManager.createException(80029); } - FsPath[] fsPaths = fsPathListWithError.getFsPaths().toArray(new FsPath[] {}); + + List fsPathList = fsPathListWithError.getFsPaths(); + // sort asc by _num.dolphin of num + ResultSetUtils.sortByNameNum(fsPathList); + FsPath[] fsPaths = fsPathList.toArray(new FsPath[] {}); + boolean isLimitDownloadSize = RESULT_SET_DOWNLOAD_IS_LIMIT.getValue(); Integer excelDownloadSize = RESULT_SET_DOWNLOAD_MAX_SIZE_EXCEL.getValue(); if (limit > 0) { @@ -855,7 +861,9 @@ public void resultsetsToExcel( response.setCharacterEncoding(StandardCharsets.UTF_8.name()); outputStream = response.getOutputStream(); // 前台传""会自动转为null - if (nullValue != null && BLANK.equalsIgnoreCase(nullValue)) nullValue = ""; + if (nullValue != null && BLANK.equalsIgnoreCase(nullValue)) { + nullValue = ""; + } fileSource = FileSource$.MODULE$.create(fsPaths, fileSystem).addParams("nullValue", nullValue); if (!FileSource$.MODULE$.isTableResultSet(fileSource)) { From 6970d3509d4e162cbb27df41a7c542c8ccaf5a01 Mon Sep 17 00:00:00 2001 From: huangKai-2323 <62878639+huangKai-2323@users.noreply.github.com> Date: Thu, 30 Mar 2023 01:16:56 +0800 Subject: [PATCH 045/261] Dev 1.3.2 fix config cache bug (#117) --- .../configuration/restful/api/ConfigurationRestfulApi.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/linkis-public-enhancements/linkis-configuration/src/main/java/org/apache/linkis/configuration/restful/api/ConfigurationRestfulApi.java b/linkis-public-enhancements/linkis-configuration/src/main/java/org/apache/linkis/configuration/restful/api/ConfigurationRestfulApi.java index fe1bd36b57..283960d5df 100644 --- a/linkis-public-enhancements/linkis-configuration/src/main/java/org/apache/linkis/configuration/restful/api/ConfigurationRestfulApi.java +++ b/linkis-public-enhancements/linkis-configuration/src/main/java/org/apache/linkis/configuration/restful/api/ConfigurationRestfulApi.java @@ -293,6 +293,10 @@ public Message saveFullTree(HttpServletRequest req, @RequestBody JsonNode json) if (tmpString.length == 2) { String engineName = tmpString[0]; String engineVersion = tmpString[1]; + logger.info( + "Config remove engine cache:engineName:{},engineVersion:{}", + engineName, + engineVersion); configurationService.clearAMCacheConf( username, Configuration.REMOVE_APPLICATION_CACHE(), From 167c212ba5ea90dd328758a3f4d8df774e31258a Mon Sep 17 00:00:00 2001 From: casionone Date: Thu, 30 Mar 2023 01:31:14 +0800 Subject: [PATCH 046/261] Revert "support spark.external.default.jars (#101)" This reverts commit faa410f8c7a0270dce26ac06bb3ae6ba302d5d4e. --- ...kSubmitProcessEngineConnLaunchBuilder.scala | 18 ++++-------------- 1 file changed, 4 insertions(+), 14 deletions(-) diff --git a/linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/launch/SparkSubmitProcessEngineConnLaunchBuilder.scala b/linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/launch/SparkSubmitProcessEngineConnLaunchBuilder.scala index 8b13e20ce1..f25a1d55b0 100644 --- a/linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/launch/SparkSubmitProcessEngineConnLaunchBuilder.scala +++ b/linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/launch/SparkSubmitProcessEngineConnLaunchBuilder.scala @@ -18,7 +18,6 @@ package org.apache.linkis.engineplugin.spark.launch import org.apache.linkis.common.conf.CommonVars -import org.apache.linkis.common.utils.Logging import org.apache.linkis.engineplugin.spark.config.SparkConfiguration.{ ENGINE_JAR, SPARK_APP_NAME, @@ -43,8 +42,7 @@ import org.apache.commons.lang3.StringUtils import scala.collection.JavaConverters._ import scala.collection.mutable.ArrayBuffer -class SparkSubmitProcessEngineConnLaunchBuilder(builder: JavaProcessEngineConnLaunchBuilder) - extends Logging { +class SparkSubmitProcessEngineConnLaunchBuilder(builder: JavaProcessEngineConnLaunchBuilder) { def getCommands( engineConnBuildRequest: EngineConnBuildRequest, @@ -56,10 +54,6 @@ class SparkSubmitProcessEngineConnLaunchBuilder(builder: JavaProcessEngineConnLa val darResource = userEngineResource.getLockedResource.asInstanceOf[DriverAndYarnResource] val properties = engineConnBuildRequest.engineConnCreationDesc.properties - if (logger.isDebugEnabled()) { - logger.debug(s"spark engine with properties:$properties") - } - val className = getValueAndRemove(properties, "className", mainClass) val driverCores = getValueAndRemove(properties, LINKIS_SPARK_DRIVER_CORES) val driverMemory = getValueAndRemove(properties, LINKIS_SPARK_DRIVER_MEMORY) @@ -69,13 +63,12 @@ class SparkSubmitProcessEngineConnLaunchBuilder(builder: JavaProcessEngineConnLa val files = getValueAndRemove(properties, "files", "").split(",").filter(isNotBlankPath) val jars = new ArrayBuffer[String]() - // jars ++= getValueAndRemove(properties, "jars", "").split(",").filter(isNotBlankPath) - var externalJars = getValueAndRemove(properties, SPARK_DEFAULT_EXTERNAL_JARS_PATH) + jars ++= getValueAndRemove(properties, "jars", "").split(",").filter(isNotBlankPath) + jars ++= getValueAndRemove(properties, SPARK_DEFAULT_EXTERNAL_JARS_PATH) .split(",") .filter(x => { - isNotBlankPath(x) + isNotBlankPath(x) && (new java.io.File(x)).isFile }) - jars ++= externalJars val pyFiles = getValueAndRemove(properties, "py-files", "").split(",").filter(isNotBlankPath) val archives = getValueAndRemove(properties, "archives", "").split(",").filter(isNotBlankPath) @@ -215,9 +208,6 @@ class SparkSubmitProcessEngineConnLaunchBuilder(builder: JavaProcessEngineConnLa ): T = { val value = commonVars.getValue(properties) properties.remove(commonVars.key) - if (logger.isDebugEnabled()) { - logger.debug(s"getValueAndRemove with key:${commonVars.key},value:$value") - } value } From d4649ce885e493323035ada3b238ce32568bd437 Mon Sep 17 00:00:00 2001 From: huangKai-2323 <62878639+huangKai-2323@users.noreply.github.com> Date: Thu, 30 Mar 2023 12:55:48 +0800 Subject: [PATCH 047/261] Dev 1.3.2 update tenant (#114) * update log Co-authored-by: Casion --- .../restful/api/TenantConfigrationRestfulApi.java | 13 ++++--------- .../service/impl/TenantConfigServiceImpl.java | 11 ++++++----- 2 files changed, 10 insertions(+), 14 deletions(-) diff --git a/linkis-public-enhancements/linkis-configuration/src/main/java/org/apache/linkis/configuration/restful/api/TenantConfigrationRestfulApi.java b/linkis-public-enhancements/linkis-configuration/src/main/java/org/apache/linkis/configuration/restful/api/TenantConfigrationRestfulApi.java index 9efbb6fd5a..cf3f9df346 100644 --- a/linkis-public-enhancements/linkis-configuration/src/main/java/org/apache/linkis/configuration/restful/api/TenantConfigrationRestfulApi.java +++ b/linkis-public-enhancements/linkis-configuration/src/main/java/org/apache/linkis/configuration/restful/api/TenantConfigrationRestfulApi.java @@ -225,9 +225,8 @@ public Message queryTenantList( public Message checkUserCreator( HttpServletRequest req, @RequestParam(value = "user", required = false) String user, - @RequestParam(value = "creator", required = false) String creator, - @RequestParam(value = "id", required = false) String id) { - Boolean checkResult = false; + @RequestParam(value = "creator", required = false) String creator) { + Boolean result = false; try { // Parameter verification if (StringUtils.isBlank(creator)) { @@ -240,15 +239,11 @@ public Message checkUserCreator( if (!Configuration.isAdmin(userName)) { return Message.error("Failed to check-user-creator,msg: only administrators can configure"); } - checkResult = tenantConfigService.isExist(user, creator); - // The incoming id represents the update. The update allows user-create exist - if (StringUtils.isNotBlank(id)) { - checkResult = !checkResult; - } + result = tenantConfigService.isExist(user, creator); } catch (ConfigurationException e) { return Message.error("Failed to check-user-creator,msg:" + e.getMessage()); } - return Message.ok().data("exist", checkResult); + return Message.ok().data("exist", result); } private void parameterVerification(TenantVo tenantVo) throws ConfigurationException { diff --git a/linkis-public-enhancements/linkis-configuration/src/main/java/org/apache/linkis/configuration/service/impl/TenantConfigServiceImpl.java b/linkis-public-enhancements/linkis-configuration/src/main/java/org/apache/linkis/configuration/service/impl/TenantConfigServiceImpl.java index 323d37ef12..bf9755a307 100644 --- a/linkis-public-enhancements/linkis-configuration/src/main/java/org/apache/linkis/configuration/service/impl/TenantConfigServiceImpl.java +++ b/linkis-public-enhancements/linkis-configuration/src/main/java/org/apache/linkis/configuration/service/impl/TenantConfigServiceImpl.java @@ -145,7 +145,7 @@ private void dataProcessing(TenantVo tenantVo) throws ConfigurationException { .forEach( map -> { String tenant = map.get("tenant").toString().toLowerCase(); - if (tenant.equals(tenantVo.getTenantValue())) { + if (tenant.equals(tenantVo.getTenantValue().toLowerCase())) { tenantResult.set(true); } }); @@ -155,20 +155,21 @@ private void dataProcessing(TenantVo tenantVo) throws ConfigurationException { throw new ConfigurationException("The ECM with the corresponding label was not found"); // The beginning of tenantValue needs to contain creator String creator = tenantVo.getCreator().toLowerCase(); - String tenantValue = tenantVo.getTenantValue().toLowerCase().split("_")[0]; - if (!creator.equals(tenantValue)) + String[] tenantArray = tenantVo.getTenantValue().toLowerCase().split("_"); + if (tenantArray.length > 1 && !creator.equals(tenantArray[0])) { throw new ConfigurationException("tenantValue should contain creator first"); + } } } @Override public Boolean isExist(String user, String creator) { - boolean result = false; + boolean result = true; Map resultMap = queryTenantList(user.toLowerCase(), creator.toLowerCase(), null, 1, 20); Object tenantList = resultMap.getOrDefault(JobRequestConstants.TOTAL_PAGE(), 0); int total = Integer.parseInt(tenantList.toString()); - if (total == 0) result = true; + if (total == 0) result = false; return result; } From 83579db47558cfb07eccd4eef2b5c145b5d9549e Mon Sep 17 00:00:00 2001 From: Casion Date: Fri, 31 Mar 2023 00:42:53 +0800 Subject: [PATCH 048/261] scala support jar (#119) * support spark.external.default.jars * Revert "support spark.external.default.jars" This reverts commit 1708dbebaebbcb878cd9fabd80427402a837a937. * remove check about isFile --- .../SparkSubmitProcessEngineConnLaunchBuilder.scala | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/launch/SparkSubmitProcessEngineConnLaunchBuilder.scala b/linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/launch/SparkSubmitProcessEngineConnLaunchBuilder.scala index f25a1d55b0..ef222865fd 100644 --- a/linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/launch/SparkSubmitProcessEngineConnLaunchBuilder.scala +++ b/linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/launch/SparkSubmitProcessEngineConnLaunchBuilder.scala @@ -18,6 +18,7 @@ package org.apache.linkis.engineplugin.spark.launch import org.apache.linkis.common.conf.CommonVars +import org.apache.linkis.common.utils.Logging import org.apache.linkis.engineplugin.spark.config.SparkConfiguration.{ ENGINE_JAR, SPARK_APP_NAME, @@ -42,7 +43,8 @@ import org.apache.commons.lang3.StringUtils import scala.collection.JavaConverters._ import scala.collection.mutable.ArrayBuffer -class SparkSubmitProcessEngineConnLaunchBuilder(builder: JavaProcessEngineConnLaunchBuilder) { +class SparkSubmitProcessEngineConnLaunchBuilder(builder: JavaProcessEngineConnLaunchBuilder) + extends Logging { def getCommands( engineConnBuildRequest: EngineConnBuildRequest, @@ -67,7 +69,13 @@ class SparkSubmitProcessEngineConnLaunchBuilder(builder: JavaProcessEngineConnLa jars ++= getValueAndRemove(properties, SPARK_DEFAULT_EXTERNAL_JARS_PATH) .split(",") .filter(x => { - isNotBlankPath(x) && (new java.io.File(x)).isFile + val isPath = isNotBlankPath(x) + // filter by isFile cannot support this case: + // The cg-linkismanager startup user is inconsistent with the engineconn startup user + + // val isFile = (new java.io.File(x)).isFile + logger.info(s"file:${x}, check isPath:${isPath}") + isPath }) val pyFiles = getValueAndRemove(properties, "py-files", "").split(",").filter(isNotBlankPath) val archives = getValueAndRemove(properties, "archives", "").split(",").filter(isNotBlankPath) From 69acd10f9c84bfed1a9f62b1eb1f5d8b349e2dfa Mon Sep 17 00:00:00 2001 From: ahaoyao <129247228+ahaoyao@users.noreply.github.com> Date: Mon, 3 Apr 2023 20:44:39 +0800 Subject: [PATCH 049/261] =?UTF-8?q?fix(common=20util)=EF=BC=9Afix=20getStr?= =?UTF-8?q?ingToMap=20function=20bug=20about=20ECResourceInfoUtils=20(#121?= =?UTF-8?q?)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../manager/am/util/ECResourceInfoUtils.java | 2 +- .../am/util/ECResourceInfoUtilsTest.java | 52 +++++++++++++++++++ 2 files changed, 53 insertions(+), 1 deletion(-) create mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/test/java/org/apache/linkis/manager/am/util/ECResourceInfoUtilsTest.java diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/util/ECResourceInfoUtils.java b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/util/ECResourceInfoUtils.java index ab926df64e..5fc8529661 100644 --- a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/util/ECResourceInfoUtils.java +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/util/ECResourceInfoUtils.java @@ -61,7 +61,7 @@ public static ResourceVo getStringToMap(String str, ECResourceInfoRecord info) { Map divermap = MapUtils.getMap(map, "driver"); resourceVo.setInstance(((Double) divermap.get("instance")).intValue()); resourceVo.setCores(((Double) divermap.get("cpu")).intValue()); - String memoryStr = String.valueOf(map.getOrDefault("memory", "0k")); + String memoryStr = String.valueOf(divermap.getOrDefault("memory", "0k")); long memorylong = 0; if (!getScientific(memoryStr)) { memorylong = ByteTimeUtils.byteStringAsBytes(memoryStr); diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/test/java/org/apache/linkis/manager/am/util/ECResourceInfoUtilsTest.java b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/test/java/org/apache/linkis/manager/am/util/ECResourceInfoUtilsTest.java new file mode 100644 index 0000000000..1d1ce10cfd --- /dev/null +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/test/java/org/apache/linkis/manager/am/util/ECResourceInfoUtilsTest.java @@ -0,0 +1,52 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.manager.am.util; + +import org.apache.linkis.common.utils.ByteTimeUtils; +import org.apache.linkis.manager.am.vo.ResourceVo; +import org.apache.linkis.manager.common.entity.persistence.ECResourceInfoRecord; +import org.apache.linkis.server.BDPJettyServerHelper; + +import java.util.HashMap; +import java.util.Map; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +/** ECResourceInfoUtilsTest */ +public class ECResourceInfoUtilsTest { + + @Test + public void testGetStringToMap() throws Exception { + ECResourceInfoRecord info = new ECResourceInfoRecord(); + info.setLabelValue("hadoop-LINKISCLI,spark-2.4.3"); + String str = + "{\"driver\":{\"instance\":1,\"memory\":\"3.0 GB\",\"cpu\":1}, \"yarn\":{\"queueName\":\"dws\",\"queueMemory\":\"2.0 GB\", \"queueCpu\":2, \"instance\":0}} "; + Map map = BDPJettyServerHelper.gson().fromJson(str, new HashMap<>().getClass()); + ResourceVo resourceVO = ECResourceInfoUtils.getStringToMap(str, info); + Map diverMap = (Map) map.get("driver"); + Assertions.assertEquals( + resourceVO.getInstance(), ((Double) diverMap.get("instance")).intValue()); + Assertions.assertEquals(resourceVO.getInstance(), 1); + Assertions.assertEquals(resourceVO.getCores(), ((Double) diverMap.get("cpu")).intValue()); + Assertions.assertEquals(resourceVO.getCores(), 1); + Assertions.assertEquals( + resourceVO.getMemory(), + ByteTimeUtils.byteStringAsBytes(String.valueOf(diverMap.getOrDefault("memory", "0k")))); + } +} From 3a25e2b9f857b256c2b2be05f7ec4da2584c74b8 Mon Sep 17 00:00:00 2001 From: huangKai-2323 <62878639+huangKai-2323@users.noreply.github.com> Date: Mon, 3 Apr 2023 21:15:24 +0800 Subject: [PATCH 050/261] Dev 1.3.2 fix stddev samp bug (#122) * fix stddev_samp bug From 0521a21764b1869f056a8811d627b538b0c820a5 Mon Sep 17 00:00:00 2001 From: casionone Date: Mon, 3 Apr 2023 21:20:45 +0800 Subject: [PATCH 051/261] revert DoubleType NaN --- .../org/apache/linkis/storage/domain/DataType.scala | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/domain/DataType.scala b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/domain/DataType.scala index 59d8860c1c..55c82abd38 100644 --- a/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/domain/DataType.scala +++ b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/domain/DataType.scala @@ -79,14 +79,7 @@ object DataType extends Logging { case IntType => if (isNumberNull(value)) null else value.toInt case LongType | BigIntType => if (isNumberNull(value)) null else value.toLong case FloatType => if (isNumberNull(value)) null else value.toFloat - case DoubleType => - if (isNumberNull(value)) { - null - } else if (value.asInstanceOf[Double].isNaN) { - "NaN" - } else { - value.toDouble - } + case DoubleType => if (isNumberNull(value)) null else value.toDouble case DecimalType => if (isNumberNull(value)) null else new JavaBigDecimal(value) case DateType => if (isNumberNull(value)) null else Date.valueOf(value) case TimestampType => From 041125e73ef9448c755b5b5db04749ba5c866aa1 Mon Sep 17 00:00:00 2001 From: casionone Date: Mon, 3 Apr 2023 21:43:06 +0800 Subject: [PATCH 052/261] fix spark resource log NPE --- .../common/utils/ByteTimeUtilsTest.java | 51 +++++++++++++++++++ .../executor/SparkEngineConnExecutor.scala | 7 +-- 2 files changed, 55 insertions(+), 3 deletions(-) create mode 100644 linkis-commons/linkis-common/src/test/java/org/apache/linkis/common/utils/ByteTimeUtilsTest.java diff --git a/linkis-commons/linkis-common/src/test/java/org/apache/linkis/common/utils/ByteTimeUtilsTest.java b/linkis-commons/linkis-common/src/test/java/org/apache/linkis/common/utils/ByteTimeUtilsTest.java new file mode 100644 index 0000000000..0d520cc05e --- /dev/null +++ b/linkis-commons/linkis-common/src/test/java/org/apache/linkis/common/utils/ByteTimeUtilsTest.java @@ -0,0 +1,51 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.common.utils; + +import static org.junit.jupiter.api.Assertions.*; + +import org.apache.linkis.common.exception.LinkisSecurityException; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +class ByteTimeUtilsTest { + + @Test + void byteStringAsBytes() { + } + + @Test + void byteStringAsKb() { + } + + @Test + void byteStringAsMb() { + } + + @Test + void byteStringAsGb() { + Long res = ByteTimeUtils.byteStringAsGb("1G"); + Assertions.assertEquals(res,1); + + Assertions.assertThrows( + NullPointerException.class, + () -> { + ByteTimeUtils.byteStringAsGb("512"); + }); + + } +} \ No newline at end of file diff --git a/linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/executor/SparkEngineConnExecutor.scala b/linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/executor/SparkEngineConnExecutor.scala index 9cd9822be3..04e735d84e 100644 --- a/linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/executor/SparkEngineConnExecutor.scala +++ b/linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/executor/SparkEngineConnExecutor.scala @@ -144,8 +144,9 @@ abstract class SparkEngineConnExecutor(val sc: SparkContext, id: Long) val sparkExecutorCores = sc.getConf.get("spark.executor.cores", "2").toInt * executorNum val sparkDriverCores = sc.getConf.get("spark.driver.cores", "1").toInt val queue = sc.getConf.get("spark.yarn.queue") - val memoryOverhead = - ByteTimeUtils.byteStringAsGb(sc.getConf.get("spark.executor.memoryOverhead", "1G")) + // with unit if set configuration with unit + // if not set sc get will get the value of spark.yarn.executor.memoryOverhead such as 512(without unit) + val memoryOverhead = sc.getConf.get("spark.executor.memoryOverhead", "1G") val sb = new StringBuilder sb.append(s"spark.executor.instances=$executorNum\n") @@ -154,7 +155,7 @@ abstract class SparkEngineConnExecutor(val sc: SparkContext, id: Long) sb.append(s"spark.executor.cores=$sparkExecutorCores\n") sb.append(s"spark.driver.cores=$sparkDriverCores\n") sb.append(s"spark.yarn.queue=$queue\n") - sb.append(s"spark.executor.memoryOverhead=${memoryOverhead}G\n") + sb.append(s"spark.executor.memoryOverhead=${memoryOverhead}\n") sb.append("\n") engineExecutionContext.appendStdout( LogUtils.generateInfo(s" Your spark job exec with configs:\n${sb.toString()}") From c602c8d9ffaaaaf178a48c62e25865bc66f9ff88 Mon Sep 17 00:00:00 2001 From: casionone Date: Tue, 4 Apr 2023 18:20:57 +0800 Subject: [PATCH 053/261] =?UTF-8?q?Revert=20"=E3=80=901.1.10=E3=80=91Fix?= =?UTF-8?q?=20the=20garbled=20code=20problem=20of=20spark=20function=20(#9?= =?UTF-8?q?3)"?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This reverts commit a18fcced --- .../linkis/engineplugin/spark/executor/SQLSession.scala | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/executor/SQLSession.scala b/linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/executor/SQLSession.scala index 9671ab6922..3291304868 100644 --- a/linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/executor/SQLSession.scala +++ b/linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/executor/SQLSession.scala @@ -214,12 +214,7 @@ object SQLSession extends Logging { .mkString("{", ",", "}") case (str: String, StringType) => str.replaceAll("\n|\t", " ") - case (double: Double, DoubleType) => - if (double.isNaN) { - "NaN" - } else { - nf.format(double) - } + case (double: Double, DoubleType) => nf.format(double) case (decimal: java.math.BigDecimal, DecimalType()) => formatDecimal(decimal) case (other: Any, tpe) => other.toString case _ => null From 9e5c0a96c9c5de4ea4eda1bdf30eddcfb0a7a2f9 Mon Sep 17 00:00:00 2001 From: huangKai-2323 <62878639+huangKai-2323@users.noreply.github.com> Date: Wed, 5 Apr 2023 22:19:04 +0800 Subject: [PATCH 054/261] Code optimization (#125) --- .../configuration/restful/api/TenantConfigrationRestfulApi.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/linkis-public-enhancements/linkis-configuration/src/main/java/org/apache/linkis/configuration/restful/api/TenantConfigrationRestfulApi.java b/linkis-public-enhancements/linkis-configuration/src/main/java/org/apache/linkis/configuration/restful/api/TenantConfigrationRestfulApi.java index cf3f9df346..9a41fe67cc 100644 --- a/linkis-public-enhancements/linkis-configuration/src/main/java/org/apache/linkis/configuration/restful/api/TenantConfigrationRestfulApi.java +++ b/linkis-public-enhancements/linkis-configuration/src/main/java/org/apache/linkis/configuration/restful/api/TenantConfigrationRestfulApi.java @@ -75,7 +75,7 @@ public Message createTenant(HttpServletRequest req, @RequestBody TenantVo tenant if (!Configuration.isAdmin(userName)) { return Message.error("Failed to create-tenant,msg: only administrators can configure"); } - if (!tenantConfigService.isExist(tenantVo.getUser(), tenantVo.getCreator())) { + if (tenantConfigService.isExist(tenantVo.getUser(), tenantVo.getCreator())) { throw new ConfigurationException("User-creator is existed"); } parameterVerification(tenantVo); From 2a5d724cc5bb28aaa0cef645c9524734e79db018 Mon Sep 17 00:00:00 2001 From: Yonghao Mei <73584269+mayinrain@users.noreply.github.com> Date: Wed, 5 Apr 2023 22:20:30 +0800 Subject: [PATCH 055/261] bug fix and feature optimize (#124) --- .../URM/module/functionManagement/index.vue | 2 +- .../apps/URM/module/udfManagement/index.vue | 2 +- .../src/apps/linkis/i18n/common/en.json | 8 +++---- .../src/apps/linkis/i18n/common/zh.json | 10 ++++----- linkis-web/src/apps/linkis/module/ECM/log.vue | 2 ++ .../module/datasourceEnv/EditForm/index.vue | 11 ++++++++-- .../linkis/module/datasourceEnv/index.vue | 17 +++++++------- .../linkis/module/gatewayAuthToken/index.vue | 2 +- .../linkis/module/ipListManagement/index.vue | 2 +- .../linkis/module/resourceManagement/log.vue | 2 ++ .../rmExternalResourceProvider/index.vue | 11 +++++----- .../apps/linkis/module/setting/setting.vue | 9 ++++---- .../module/tenantTagManagement/index.vue | 22 +++++++++---------- .../src/apps/linkis/module/udfTree/index.vue | 2 +- .../src/apps/linkis/view/linkis/index.vue | 10 ++++----- 15 files changed, 62 insertions(+), 50 deletions(-) diff --git a/linkis-web/src/apps/URM/module/functionManagement/index.vue b/linkis-web/src/apps/URM/module/functionManagement/index.vue index c756513faf..389b1003c3 100644 --- a/linkis-web/src/apps/URM/module/functionManagement/index.vue +++ b/linkis-web/src/apps/URM/module/functionManagement/index.vue @@ -573,7 +573,7 @@ export default { if (!args.row) return this.$Modal.confirm({ title: this.$t('message.linkis.modal.modalTitle'), - content: this.$t('message.linkis.modal.modalDelete', {envName: args.row.udfName}), + content: this.$t('message.linkis.modal.modalDelete', {name: args.row.udfName}), onOk: ()=>{ api .fetch(`/udf/delete/${args.row.id}`, {}, 'post') diff --git a/linkis-web/src/apps/URM/module/udfManagement/index.vue b/linkis-web/src/apps/URM/module/udfManagement/index.vue index da73981ae0..870b6ddf2c 100644 --- a/linkis-web/src/apps/URM/module/udfManagement/index.vue +++ b/linkis-web/src/apps/URM/module/udfManagement/index.vue @@ -567,7 +567,7 @@ export default { if (!args.row) return this.$Modal.confirm({ title: this.$t('message.linkis.modal.modalTitle'), - content: this.$t('message.linkis.modal.modalDelete', {envName: args.row.udfName}), + content: this.$t('message.linkis.modal.modalDelete', {name: args.row.udfName}), onOk: ()=>{ api.fetch(`/udf/delete/${args.row.id}`, {}, 'post') .then(() => { diff --git a/linkis-web/src/apps/linkis/i18n/common/en.json b/linkis-web/src/apps/linkis/i18n/common/en.json index 845b6fe03a..470d9c8ca3 100644 --- a/linkis-web/src/apps/linkis/i18n/common/en.json +++ b/linkis-web/src/apps/linkis/i18n/common/en.json @@ -398,7 +398,7 @@ "addSuccessful": "Successfully Addded", "editSuccessful": "Successfully edited", "confirmDel": "Confirm Deletion", - "isConfirmDel": "Are you sure you want to delete this tag" + "isConfirmDel": "Are you sure you want to delete this tag({name})" }, "ipListManagement": { "userName": "User Name", @@ -433,7 +433,7 @@ "addSuccessful": "Successfully Addded", "editSuccessful": "Successfully edited", "confirmDel": "Confirm Deletion", - "isConfirmDel": "Are you sure you want to delete this tag" + "isConfirmDel": "Are you sure you want to delete this tag({name})" }, "codeQuery": { "executionCode": "Execution Code", @@ -468,8 +468,8 @@ "cancel": "Cancel", "modalTitle": "Info", "modalFormat": "Confirm deleting the {0} record?", - "modalDelete1": "Confirm that the record [{username}] should be deleted?", - "modalDelete": "Confirm that the record [{envName}] should be deleted?", + "modalDelete1": "Confirm that the record [{name}] should be deleted?", + "modalDelete": "Confirm that the record [{name}] should be deleted?", "modalDeleteSuccess": "Successfully delete", "modalDeleteFail": "Fail to delete", "modalAddSuccess": "Successfully added", diff --git a/linkis-web/src/apps/linkis/i18n/common/zh.json b/linkis-web/src/apps/linkis/i18n/common/zh.json index cc04f4ab61..49950288ca 100644 --- a/linkis-web/src/apps/linkis/i18n/common/zh.json +++ b/linkis-web/src/apps/linkis/i18n/common/zh.json @@ -194,7 +194,7 @@ }, "modal": { "modalTitle": "提示信息", - "modalDelete": "确认是否删除[{envName}]项?", + "modalDelete": "确认是否删除[{name}]项?", "modalDeleteRecord": "确认是否删除该记录?", "modalDeleteTask": "确认是否停止当前选择任务?", "modalDeleteInstance": "确认是否停止当前选择实例?", @@ -396,7 +396,7 @@ "userIsExisted": "用户标签已存在", "addSuccessful": "添加成功", "confirmDel": "确认删除", - "isConfirmDel": "确定要删除这条数据吗" + "isConfirmDel": "确定要删除这条数据吗({name})" }, "ipListManagement": { "userName": "用户名", @@ -430,7 +430,7 @@ "userIsExisted": "用户标签已存在", "addSuccessful": "添加成功", "confirmDel": "确认删除", - "isConfirmDel": "确定要删除这条数据吗" + "isConfirmDel": "确定要删除这条数据吗({name})" }, "basedataManagement": { "add": "新增", @@ -446,8 +446,8 @@ "cancel": "取消", "modalTitle": "提示信息", "modalFormat": "确定删除 {0} 这条记录?", - "modalDelete1": "确认是否删除[{username}]该记录?", - "modalDelete": "确认是否删除[{envName}]该记录?", + "modalDelete1": "确认是否删除[{name}]该记录?", + "modalDelete": "确认是否删除[{name}]该记录?", "modalDeleteSuccess": "删除成功", "modalDeleteFail": "删除失败", "modalAddSuccess": "添加成功", diff --git a/linkis-web/src/apps/linkis/module/ECM/log.vue b/linkis-web/src/apps/linkis/module/ECM/log.vue index d7a54586ca..2c2ffc13fb 100644 --- a/linkis-web/src/apps/linkis/module/ECM/log.vue +++ b/linkis-web/src/apps/linkis/module/ECM/log.vue @@ -20,6 +20,8 @@ + + diff --git a/linkis-web/src/apps/linkis/module/datasourceEnv/EditForm/index.vue b/linkis-web/src/apps/linkis/module/datasourceEnv/EditForm/index.vue index 71f02e4405..ca19bd336c 100644 --- a/linkis-web/src/apps/linkis/module/datasourceEnv/EditForm/index.vue +++ b/linkis-web/src/apps/linkis/module/datasourceEnv/EditForm/index.vue @@ -239,13 +239,20 @@ export default { }) }, hiddenHandler (newV) { + let dataSourceTypeName = ''; + for(let i = 0;i < this.typeOptions.length; i++) { + if (this.typeOptions[i].value === newV.datasourceTypeId) { + dataSourceTypeName = this.typeOptions[i].label + } + } // radio - this.rule[4].hidden = !(newV.datasourceTypeId === 4); + this.rule[4].hidden = !(['hive', 'kafka'].includes(dataSourceTypeName)) + // keytab value this.rule[6].hidden = !newV.keytab; // upload this.rule[5].hidden = !this.rule[6].hidden; - if (!newV.hasKeyTab || newV.datasourceTypeId !== 4){ + if (!newV.hasKeyTab || !['hive', 'kafka'].includes(dataSourceTypeName)){ this.rule[5].hidden = true; this.rule[6].hidden = true; this.rule[8].hidden = true; diff --git a/linkis-web/src/apps/linkis/module/datasourceEnv/index.vue b/linkis-web/src/apps/linkis/module/datasourceEnv/index.vue index 5a2666f153..aabd96ab8c 100644 --- a/linkis-web/src/apps/linkis/module/datasourceEnv/index.vue +++ b/linkis-web/src/apps/linkis/module/datasourceEnv/index.vue @@ -70,7 +70,7 @@ - +
@@ -204,7 +204,7 @@ export default { init() { this.load(); }, - load() { + async load() { let params = { searchName: this.searchName, currentPage: this.page.pageNow, @@ -212,7 +212,7 @@ export default { } getAllEnv().then((res) => { let options = [...res.typeList].sort((a, b) => a.id - b.id) - .map(item => { return {value: +item.id, label: item.name, disabled: ![2, 4].includes(+item.id)}}) + .map(item => { return {value: +item.id, label: item.name, disabled: !['hive', 'kafka'].includes(item.name)}}) this.datasourceTypeOptions= options // 获取列表 getList(params).then((data) => { @@ -223,7 +223,7 @@ export default { let filter = options.filter(optionsItem=>{ return optionsItem.value === item.datasourceTypeId }) - item.name = filter[0].label + item.name = filter[0]?.label || ''; }) }) }) @@ -287,8 +287,8 @@ export default { } this.modalEditData.hasKeyTab = false; }, - onModalOk(){ - this.$refs.editForm.formModel.submit((formData)=>{ + async onModalOk(){ + this.$refs.editForm.formModel.submit(async (formData)=>{ if (!('parameter' in formData)) { formData['parameter'] = {} } @@ -313,7 +313,7 @@ export default { if('keytab' in formData) delete formData['keytab']; if('uris' in formData) delete formData['uris']; if(this.modalAddMode=='add') { - add(formData).then((data)=>{ + await add(formData).then((data)=>{ //window.console.log(data) if(data.result) { this.$Message.success({ @@ -328,7 +328,7 @@ export default { } }) }else { - edit(formData).then((data)=>{ + await edit(formData).then((data)=>{ //window.console.log(data) if(data.result) { this.$Message.success({ @@ -347,6 +347,7 @@ export default { window.console.log(formData); this.modalLoading=false this.modalShow = false + await this.load(); }) }, onModalCancel(){ diff --git a/linkis-web/src/apps/linkis/module/gatewayAuthToken/index.vue b/linkis-web/src/apps/linkis/module/gatewayAuthToken/index.vue index 19197f9c50..b99badcb48 100644 --- a/linkis-web/src/apps/linkis/module/gatewayAuthToken/index.vue +++ b/linkis-web/src/apps/linkis/module/gatewayAuthToken/index.vue @@ -234,7 +234,7 @@ export default { onTableDelete(row){ this.$Modal.confirm({ title: this.$t('message.linkis.basedataManagement.modal.modalTitle'), - content: this.$t('message.linkis.basedataManagement.modal.modalDelete', {envName: row.tokenName}), + content: this.$t('message.linkis.basedataManagement.modal.modalDelete', {name: row.tokenName}), onOk: ()=>{ let params = { id: row.id diff --git a/linkis-web/src/apps/linkis/module/ipListManagement/index.vue b/linkis-web/src/apps/linkis/module/ipListManagement/index.vue index 770795ce2b..e411dc7817 100644 --- a/linkis-web/src/apps/linkis/module/ipListManagement/index.vue +++ b/linkis-web/src/apps/linkis/module/ipListManagement/index.vue @@ -389,7 +389,7 @@ export default { delete(data) { this.$Modal.confirm({ title: this.$t('message.linkis.ipListManagement.confirmDel'), - content: this.$t('message.linkis.ipListManagement.isConfirmDel'), + content: this.$t('message.linkis.ipListManagement.isConfirmDel', {name: `id:${data.id}`}), onOk: async () => { await this.confirmDelete(data); await this.getTableData(); diff --git a/linkis-web/src/apps/linkis/module/resourceManagement/log.vue b/linkis-web/src/apps/linkis/module/resourceManagement/log.vue index acf96cbb49..5fef5d8f49 100644 --- a/linkis-web/src/apps/linkis/module/resourceManagement/log.vue +++ b/linkis-web/src/apps/linkis/module/resourceManagement/log.vue @@ -20,6 +20,8 @@ + + diff --git a/linkis-web/src/apps/linkis/module/rmExternalResourceProvider/index.vue b/linkis-web/src/apps/linkis/module/rmExternalResourceProvider/index.vue index 98181836d3..9e0f610800 100644 --- a/linkis-web/src/apps/linkis/module/rmExternalResourceProvider/index.vue +++ b/linkis-web/src/apps/linkis/module/rmExternalResourceProvider/index.vue @@ -160,7 +160,7 @@ export default { init() { window.console.log(this.$route.query.isSkip); }, - load() { + async load() { let params = { searchName: this.searchName, currentPage: this.page.pageNow, @@ -219,12 +219,12 @@ export default { }) }, - onModalOk(){ - this.$refs.errorCodeForm.formModel.submit((formData)=>{ + async onModalOk(){ + this.$refs.errorCodeForm.formModel.submit(async (formData)=>{ this.modalLoading = true formData.config = JSON.stringify(formData.config) if(this.modalAddMode=='add') { - add(formData).then((data)=>{ + await add(formData).then((data)=>{ window.console.log(data) if(data.result) { this.$Message.success({ @@ -239,7 +239,7 @@ export default { } }) }else { - edit(formData).then((data)=>{ + await edit(formData).then((data)=>{ window.console.log(data) if(data.result) { this.$Message.success({ @@ -257,6 +257,7 @@ export default { } this.modalLoading=false this.modalShow = false + await this.load() }) }, onModalCancel(){ diff --git a/linkis-web/src/apps/linkis/module/setting/setting.vue b/linkis-web/src/apps/linkis/module/setting/setting.vue index ce1ae5aeef..3fed0420fe 100644 --- a/linkis-web/src/apps/linkis/module/setting/setting.vue +++ b/linkis-web/src/apps/linkis/module/setting/setting.vue @@ -565,11 +565,10 @@ export default { //Find the data in the menuList of the tab you clicked to delete(找到点击删除的tab 在menuList里的数据) let menuItem = this.menuList.find((item) => item.categoryName === name); //Determine whether the current active tab is the same as the tab currently clicked to delete(判断当前的活动tab是否和当前点击删除的tab一样) - if (this.currentTabName === name) { - let ItemIndex = this.menuList.findIndex( - (item) => item.categoryName === this.currentTabName - ); - this.menuList.splice(ItemIndex, 1); + const activeIndex = this.menuList.findIndex((item) => item.categoryName === this.currentTabName) + 1; + const curIndex = this.menuList.findIndex((item) => item.categoryName === name); + if (curIndex === activeIndex) { + this.menuList.splice(curIndex, 1); api .fetch( "/configuration/deleteCategory", diff --git a/linkis-web/src/apps/linkis/module/tenantTagManagement/index.vue b/linkis-web/src/apps/linkis/module/tenantTagManagement/index.vue index fccbe7f35c..6c49f1a5db 100644 --- a/linkis-web/src/apps/linkis/module/tenantTagManagement/index.vue +++ b/linkis-web/src/apps/linkis/module/tenantTagManagement/index.vue @@ -223,7 +223,7 @@ export default { ], creator: [ {required: true, message: this.$t('message.linkis.tenantTagManagement.notEmpty'), trigger: 'blur'}, - {pattern: /^[0-9a-zA-Z_]+$/, message: this.$t('message.linkis.tenantTagManagement.contentError1'), type: 'string'} + {pattern: /^[0-9a-zA-Z_\*]+$/, message: this.$t('message.linkis.tenantTagManagement.contentError'), type: 'string'} ], tenantValue: [ {required: true, message: this.$t('message.linkis.tenantTagManagement.notEmpty'), trigger: 'blur'}, @@ -313,22 +313,22 @@ export default { if(this.isRequesting) return; this.$refs.createTenantForm.validate(async (valid) => { if(valid) { - const {user, creator} = this.modalData; + const { user, creator, id } = this.modalData; if(this.mode === 'edit' && user === this.editData.user && creator === this.editData.creator) { this.tagIsExist = false; return; } try { - this.isRequesting = true - await api.fetch("/configuration/tenant-mapping/check-user-creator", - { - user, - creator - }, "get").then((res) => { - if (res.exist) { + this.isRequesting = true; + const checkBody = {user, creator}; + if(this.mode === 'edit') { + checkBody.id = id; + } + await api.fetch("/configuration/tenant-mapping/check-user-creator", checkBody, "get").then((res) => { + if (!res.exist) { this.$Message.error(this.$t('message.linkis.tenantTagManagement.userIsExisted')) } - this.tagIsExist = res.exist; + this.tagIsExist = !res.exist; }) this.isRequesting = false } catch (err) { @@ -404,7 +404,7 @@ export default { delete(data) { this.$Modal.confirm({ title: this.$t('message.linkis.tenantTagManagement.confirmDel'), - content: this.$t('message.linkis.tenantTagManagement.isConfirmDel'), + content: this.$t('message.linkis.tenantTagManagement.isConfirmDel', {name: `id:${data.id}`}), onOk: async () => { await this.confirmDelete(data); await this.getTableData(); diff --git a/linkis-web/src/apps/linkis/module/udfTree/index.vue b/linkis-web/src/apps/linkis/module/udfTree/index.vue index 0ef454f9ce..7a12abf941 100644 --- a/linkis-web/src/apps/linkis/module/udfTree/index.vue +++ b/linkis-web/src/apps/linkis/module/udfTree/index.vue @@ -224,7 +224,7 @@ export default { this.$Modal.confirm({ title: this.$t('message.linkis.basedataManagement.modal.modalTitle'), - content: this.$t('message.linkis.basedataManagement.modal.modalDelete'), + content: this.$t('message.linkis.basedataManagement.modal.modalDelete', {name: row.name}), onOk: ()=>{ let params = { id: row.id diff --git a/linkis-web/src/apps/linkis/view/linkis/index.vue b/linkis-web/src/apps/linkis/view/linkis/index.vue index 88e5088af5..6749d85cdc 100644 --- a/linkis-web/src/apps/linkis/view/linkis/index.vue +++ b/linkis-web/src/apps/linkis/view/linkis/index.vue @@ -51,7 +51,7 @@ v-for="(item3, index3) in (item.key === '1-9' ? urmSideNavList.children : item.key === '1-8' ?datasourceNavList.children:basedataNavList.children)" :key="index3" @on-click="clickToRoute">
+ v-if="isLogAdmin ? true : item3.key === '1-8-1' || item3.key === '1-9-2' || item3.key === '1-9-1'">
@@ -180,7 +180,7 @@ export default { ), showSubMenu: true, }, - // { key: '1-11', name: this.$t('message.linkis.sideNavList.function.children.codeQuery'), path: '/console/codeQuery' }, + { key: '1-11', name: this.$t('message.linkis.sideNavList.function.children.codeQuery'), path: '/console/codeQuery' }, ], }, datasourceNavList: { @@ -203,7 +203,7 @@ export default { ), path: '/console/datasourceEnv', }, - // {key: '1-8-3', name: this.$t('message.linkis.sideNavList.function.children.datasourceType'), path: '/console/datasourceType' }, + {key: '1-8-3', name: this.$t('message.linkis.sideNavList.function.children.datasourceType'), path: '/console/datasourceType' }, // {key: '1-8-4', name: this.$t('message.linkis.sideNavList.function.children.datasourceAccess'), path: '/console/datasourceAccess' }, { key: '1-8-5', @@ -284,8 +284,8 @@ export default { ), path: '/console/urm/functionManagement', }, - // {key: '1-9-3', name: this.$t('message.linkis.sideNavList.function.children.udfManager'), path: '/console/udfManager' }, - // {key: '1-9-4', name: this.$t('message.linkis.sideNavList.function.children.udfTree'), path: '/console/udfTree' }, + {key: '1-9-3', name: this.$t('message.linkis.sideNavList.function.children.udfManager'), path: '/console/udfManager' }, + {key: '1-9-4', name: this.$t('message.linkis.sideNavList.function.children.udfTree'), path: '/console/udfTree' }, ], }, breadcrumbSecondName: this.$t( From 141babf802ba6f1721706b458a00046b8d5f2e13 Mon Sep 17 00:00:00 2001 From: Yonghao Mei <73584269+mayinrain@users.noreply.github.com> Date: Thu, 6 Apr 2023 16:38:02 +0800 Subject: [PATCH 056/261] add 1.1.10 webank web (#127) * bug fix * modify package.json --- linkis-web/package.json | 37 ++++++++++--------- .../linkis/components/variable/index.scss | 1 + .../src/apps/linkis/i18n/common/en.json | 2 +- .../src/apps/linkis/module/ECM/engineConn.vue | 3 +- linkis-web/src/apps/linkis/module/ECM/log.vue | 6 ++- .../linkis/module/datasourceAccess/index.vue | 1 + .../linkis/module/datasourceEnv/index.vue | 2 +- .../linkis/module/datasourceType/index.vue | 1 + .../linkis/module/datasourceTypeKey/index.vue | 2 +- .../apps/linkis/module/errorCode/index.vue | 3 +- .../linkis/module/gatewayAuthToken/index.vue | 2 +- .../globalHistoryManagement/viewHistory.vue | 1 + .../module/resourceManagement/engineConn.vue | 1 + .../linkis/module/resourceManagement/log.vue | 6 ++- .../rmExternalResourceProvider/index.vue | 2 +- .../module/tenantTagManagement/index.vue | 9 +++-- .../apps/linkis/module/udfManager/index.vue | 3 +- .../src/apps/linkis/module/udfTree/index.vue | 13 ++++--- linkis-web/src/common/i18n/en.json | 1 + linkis-web/src/common/i18n/zh.json | 1 + .../consoleComponent/resultSetList.vue | 6 ++- .../components/consoleComponent/toolbar.vue | 7 ++++ 22 files changed, 67 insertions(+), 43 deletions(-) diff --git a/linkis-web/package.json b/linkis-web/package.json index 0b1b838556..809a551561 100644 --- a/linkis-web/package.json +++ b/linkis-web/package.json @@ -18,13 +18,12 @@ }, "lint-staged": { "src/**/*.{js,vue}": [ - "vue-cli-service lint --no-fix", - "git add" + "vue-cli-service lint --no-fix" ] }, "dependencies": { "@form-create/iview": "2.5.27", - "axios": "^0.21.4", + "axios": "0.21.4", "babel-polyfill": "6.26.0", "core-js": "3.27.2", "dexie": "3.2.3", @@ -34,17 +33,18 @@ "highlight.js": "10.7.0", "iview": "3.5.4", "jsencrypt": "3.2.1", - "lodash": "^4.17.21", + "lodash": "4.17.21", "md5": "2.3.0", "mitt": "1.2.0", - "moment": "^2.29.4", + "moment": "2.29.4", "monaco-editor": "0.30.1", "object-to-formdata": "4.2.2", - "path-browserify": "^1.0.1", - "qs": "^6.11.0", + "path-browserify": "1.0.1", + "postcss": "8.4.21", + "qs": "6.11.0", "reconnecting-websocket": "4.4.0", "sql-formatter": "2.3.3", - "svgo": "^3.0.2", + "svgo": "3.0.2", "v-jsoneditor": "1.4.5", "vue": "2.6.12", "vue-i18n": "8.22.1", @@ -55,28 +55,29 @@ }, "devDependencies": { "@intlify/vue-i18n-loader": "1.0.0", - "@vue/cli-plugin-babel": "^5.0.8", - "@vue/cli-plugin-eslint": "^5.0.8", - "@vue/cli-service": "^5.0.8", + "@vue/cli-plugin-babel": "5.0.8", + "@vue/cli-plugin-eslint": "5.0.8", + "@vue/cli-service": "5.0.8", "@vue/eslint-config-standard": "4.0.0", "archiver": "3.1.1", + "autoprefixer": "10.4.14", "babel-eslint": "10.1.0", - "copy-webpack-plugin": "^9.1.0", + "copy-webpack-plugin": "9.1.0", "csp-html-webpack-plugin": "5.1.0", - "filemanager-webpack-plugin": "^7.0.0", + "filemanager-webpack-plugin": "7.0.0", "husky": "1.3.1", - "lint-staged": "^13.1.1", + "lint-staged": "13.1.1", "material-design-icons": "3.0.1", "monaco-editor-webpack-plugin": "6.0.0", - "node-sass": "^8.0.0", - "npm-force-resolutions": "^0.0.10", - "sass-loader": "^10.4.1", + "node-sass": "8.0.0", + "npm-force-resolutions": "0.0.10", + "sass-loader": "10.4.1", "svg-sprite-loader": "6.0.0", "vue-cli-plugin-mockjs": "0.1.3", "vue-template-compiler": "2.6.12", "webpack-virtual-modules": "0.3.2" }, "resolutions": { - "postcss": "7.0.36" + "postcss": "8.4.21" } } diff --git a/linkis-web/src/apps/linkis/components/variable/index.scss b/linkis-web/src/apps/linkis/components/variable/index.scss index 643948e8b7..9140e16a35 100644 --- a/linkis-web/src/apps/linkis/components/variable/index.scss +++ b/linkis-web/src/apps/linkis/components/variable/index.scss @@ -50,6 +50,7 @@ line-height: 24px; display: flex; align-items: center; + position: relative; @media only screen and (max-width: 1480px){ .we-variable-content-label-group { display: flex !important; diff --git a/linkis-web/src/apps/linkis/i18n/common/en.json b/linkis-web/src/apps/linkis/i18n/common/en.json index 470d9c8ca3..aba5ff0c61 100644 --- a/linkis-web/src/apps/linkis/i18n/common/en.json +++ b/linkis-web/src/apps/linkis/i18n/common/en.json @@ -194,7 +194,7 @@ }, "modal": { "modalTitle": "Prompt message ", - "modalDelete": "Confirm whether to delete the [{envName} option?", + "modalDelete": "Confirm whether to delete the [{name} option?", "modalDeleteRecord": "Confirm whether to delete the record?", "modalDeleteTask": "Confirm whether to stop the selected task?", "modalDeleteInstance": "Determines whether to stop the current selected instance?", diff --git a/linkis-web/src/apps/linkis/module/ECM/engineConn.vue b/linkis-web/src/apps/linkis/module/ECM/engineConn.vue index d0004694be..5fd9c221cc 100644 --- a/linkis-web/src/apps/linkis/module/ECM/engineConn.vue +++ b/linkis-web/src/apps/linkis/module/ECM/engineConn.vue @@ -226,7 +226,8 @@ export default { this.$refs.logPanel.getLogs(0, { emInstance: params.row.emInstance, instance: params.row.instance, - applicationName: params.row.applicationName + applicationName: params.row.applicationName, + engineType: params.row.engineType, }) } } diff --git a/linkis-web/src/apps/linkis/module/ECM/log.vue b/linkis-web/src/apps/linkis/module/ECM/log.vue index 2c2ffc13fb..c186edaf40 100644 --- a/linkis-web/src/apps/linkis/module/ECM/log.vue +++ b/linkis-web/src/apps/linkis/module/ECM/log.vue @@ -21,7 +21,7 @@ - + @@ -57,7 +57,8 @@ export default { }, scriptViewState: { bottomContentHeight: window.innerHeight - 353 - } + }, + engineType: '', }; }, computed: { @@ -82,6 +83,7 @@ export default { async getLogs(fromLine, param) { if (param) { this.param = param + this.engineType = param.engineType } if (this.param) { const params = { diff --git a/linkis-web/src/apps/linkis/module/datasourceAccess/index.vue b/linkis-web/src/apps/linkis/module/datasourceAccess/index.vue index 8386ba9399..33f30de9bb 100644 --- a/linkis-web/src/apps/linkis/module/datasourceAccess/index.vue +++ b/linkis-web/src/apps/linkis/module/datasourceAccess/index.vue @@ -242,6 +242,7 @@ export default { duration: 3, content: "添加成功" }) + this.load() }else{ this.$Message.success({ duration: 3, diff --git a/linkis-web/src/apps/linkis/module/datasourceEnv/index.vue b/linkis-web/src/apps/linkis/module/datasourceEnv/index.vue index aabd96ab8c..553817e8c5 100644 --- a/linkis-web/src/apps/linkis/module/datasourceEnv/index.vue +++ b/linkis-web/src/apps/linkis/module/datasourceEnv/index.vue @@ -320,6 +320,7 @@ export default { duration: 3, content: this.$t('message.linkis.basedataManagement.modal.modalAddSuccess') }) + this.load(); }else{ this.$Message.success({ duration: 3, @@ -347,7 +348,6 @@ export default { window.console.log(formData); this.modalLoading=false this.modalShow = false - await this.load(); }) }, onModalCancel(){ diff --git a/linkis-web/src/apps/linkis/module/datasourceType/index.vue b/linkis-web/src/apps/linkis/module/datasourceType/index.vue index 5a78fd92be..5dbb8bf6f7 100644 --- a/linkis-web/src/apps/linkis/module/datasourceType/index.vue +++ b/linkis-web/src/apps/linkis/module/datasourceType/index.vue @@ -247,6 +247,7 @@ export default { duration: 3, content: this.$t('message.linkis.basedataManagement.modal.modalAddSuccess') }) + this.load() }else{ this.$Message.success({ duration: 3, diff --git a/linkis-web/src/apps/linkis/module/datasourceTypeKey/index.vue b/linkis-web/src/apps/linkis/module/datasourceTypeKey/index.vue index 42823c657a..124faa352e 100644 --- a/linkis-web/src/apps/linkis/module/datasourceTypeKey/index.vue +++ b/linkis-web/src/apps/linkis/module/datasourceTypeKey/index.vue @@ -295,13 +295,13 @@ export default { duration: 3, content: this.$t('message.linkis.basedataManagement.modal.modalAddSuccess') }) + this.load() }else{ this.$Message.success({ duration: 3, content: this.$t('message.linkis.basedataManagement.modal.modalAddFail') }) } - this.load() }) }else { edit(formData).then((data)=>{ diff --git a/linkis-web/src/apps/linkis/module/errorCode/index.vue b/linkis-web/src/apps/linkis/module/errorCode/index.vue index 332bb3405a..cd04394995 100644 --- a/linkis-web/src/apps/linkis/module/errorCode/index.vue +++ b/linkis-web/src/apps/linkis/module/errorCode/index.vue @@ -222,13 +222,13 @@ export default { duration: 3, content: this.$t('message.linkis.basedataManagement.modal.modalAddSuccess') }) + this.load() }else{ this.$Message.success({ duration: 3, content: this.$t('message.linkis.basedataManagement.modal.modalAddFail') }) } - this.load() }) }else { edit(formData).then((data)=>{ @@ -246,7 +246,6 @@ export default { }) } }) - this.load() } this.modalLoading=false this.modalShow = false diff --git a/linkis-web/src/apps/linkis/module/gatewayAuthToken/index.vue b/linkis-web/src/apps/linkis/module/gatewayAuthToken/index.vue index b99badcb48..91a3eaaf00 100644 --- a/linkis-web/src/apps/linkis/module/gatewayAuthToken/index.vue +++ b/linkis-web/src/apps/linkis/module/gatewayAuthToken/index.vue @@ -268,13 +268,13 @@ export default { duration: 3, content: this.$t('message.linkis.basedataManagement.modal.modalAddSuccess') }) + this.load() }else{ this.$Message.success({ duration: 3, content: this.$t('message.linkis.basedataManagement.modal.modalAddFail') }) } - this.load() }) }else { edit(formData).then((data)=>{ diff --git a/linkis-web/src/apps/linkis/module/globalHistoryManagement/viewHistory.vue b/linkis-web/src/apps/linkis/module/globalHistoryManagement/viewHistory.vue index 069068c74d..8742188b90 100644 --- a/linkis-web/src/apps/linkis/module/globalHistoryManagement/viewHistory.vue +++ b/linkis-web/src/apps/linkis/module/globalHistoryManagement/viewHistory.vue @@ -286,6 +286,7 @@ export default { let jobhistory = await api.fetch(`/jobhistory/${jobId}/get`, 'get') const option = jobhistory.task this.jobhistoryTask = option + this.script.runType = option.runType if (!jobhistory.task.logPath) { const errCode = jobhistory.task.errCode ? `\n${this.$t('message.linkis.errorCode')}：${ diff --git a/linkis-web/src/apps/linkis/module/resourceManagement/engineConn.vue b/linkis-web/src/apps/linkis/module/resourceManagement/engineConn.vue index 8e71ef8645..f8da125f7c 100644 --- a/linkis-web/src/apps/linkis/module/resourceManagement/engineConn.vue +++ b/linkis-web/src/apps/linkis/module/resourceManagement/engineConn.vue @@ -203,6 +203,7 @@ export default { emInstance: params.row.ecmInstance, instance: params.row.serviceInstance, ticketId: params.row.ticketId, + engineType: params.row.engineType, logDirSuffix: params.row.logDirSuffix, }) } diff --git a/linkis-web/src/apps/linkis/module/resourceManagement/log.vue b/linkis-web/src/apps/linkis/module/resourceManagement/log.vue index 5fef5d8f49..39f79b0035 100644 --- a/linkis-web/src/apps/linkis/module/resourceManagement/log.vue +++ b/linkis-web/src/apps/linkis/module/resourceManagement/log.vue @@ -21,7 +21,7 @@ - + @@ -57,7 +57,8 @@ export default { }, scriptViewState: { bottomContentHeight: window.innerHeight - 353 - } + }, + engineType: '', }; }, computed: { @@ -82,6 +83,7 @@ export default { async getLogs(fromLine, param) { if (param) { this.param = param + this.engineType = param.engineType } if (this.param) { const params = { diff --git a/linkis-web/src/apps/linkis/module/rmExternalResourceProvider/index.vue b/linkis-web/src/apps/linkis/module/rmExternalResourceProvider/index.vue index 9e0f610800..9e2f175e95 100644 --- a/linkis-web/src/apps/linkis/module/rmExternalResourceProvider/index.vue +++ b/linkis-web/src/apps/linkis/module/rmExternalResourceProvider/index.vue @@ -231,6 +231,7 @@ export default { duration: 3, content: this.$t('message.linkis.basedataManagement.modal.modalAddSuccess') }) + this.load() }else{ this.$Message.success({ duration: 3, @@ -257,7 +258,6 @@ export default { } this.modalLoading=false this.modalShow = false - await this.load() }) }, onModalCancel(){ diff --git a/linkis-web/src/apps/linkis/module/tenantTagManagement/index.vue b/linkis-web/src/apps/linkis/module/tenantTagManagement/index.vue index 6c49f1a5db..a7aab77405 100644 --- a/linkis-web/src/apps/linkis/module/tenantTagManagement/index.vue +++ b/linkis-web/src/apps/linkis/module/tenantTagManagement/index.vue @@ -106,7 +106,7 @@
{{ $t('message.linkis.tenantTagManagement.yourTagMapping') }} - +
@@ -325,7 +325,7 @@ export default { checkBody.id = id; } await api.fetch("/configuration/tenant-mapping/check-user-creator", checkBody, "get").then((res) => { - if (!res.exist) { + if (res.exist) { this.$Message.error(this.$t('message.linkis.tenantTagManagement.userIsExisted')) } this.tagIsExist = !res.exist; @@ -399,6 +399,7 @@ export default { id, user, creator, tenantValue, bussinessUser, desc }; this.showCreateModal = true; + this.tagIsExist = false; this.mode = 'edit'; }, delete(data) { @@ -422,7 +423,9 @@ export default { } }, async handleChange() { - this.tagIsExist = true; + if(this.mode !== 'edit') { + this.tagIsExist = true; + } }, async changePage(val) { this.page.pageNow = val; diff --git a/linkis-web/src/apps/linkis/module/udfManager/index.vue b/linkis-web/src/apps/linkis/module/udfManager/index.vue index 66319cd16c..b1f425acbe 100644 --- a/linkis-web/src/apps/linkis/module/udfManager/index.vue +++ b/linkis-web/src/apps/linkis/module/udfManager/index.vue @@ -211,13 +211,13 @@ export default { duration: 3, content: this.$t('message.linkis.basedataManagement.modal.modalAddSuccess') }) + this.load() }else{ this.$Message.success({ duration: 3, content: this.$t('message.linkis.basedataManagement.modal.modalAddFail') }) } - this.load() }) }else { edit(formData).then((data)=>{ @@ -234,7 +234,6 @@ export default { content: this.$t('message.linkis.basedataManagement.modal.modalEditFail') }) } - this.load() }) } this.modalLoading=false diff --git a/linkis-web/src/apps/linkis/module/udfTree/index.vue b/linkis-web/src/apps/linkis/module/udfTree/index.vue index 7a12abf941..214ed2fff0 100644 --- a/linkis-web/src/apps/linkis/module/udfTree/index.vue +++ b/linkis-web/src/apps/linkis/module/udfTree/index.vue @@ -91,7 +91,7 @@ import mixin from '@/common/service/mixin'; import ErrorCodeForm from './EditForm/index' import {add, del, edit, getList} from "./service"; -import {formatDate} from "iview/src/components/date-picker/util"; +// import {formatDate} from "iview/src/components/date-picker/util"; export default { mixins: [mixin], components: {ErrorCodeForm}, @@ -151,7 +151,7 @@ export default { align: 'center', render: (h,params)=>{ return h('div', - formatDate(new Date(params.row.createTime),'yyyy-MM-dd hh:mm:ss') + new Date(params.row.createTime).toLocaleString() ) } }, @@ -163,7 +163,7 @@ export default { align: 'center', render: (h,params)=>{ return h('div', - formatDate(new Date(params.row.updateTime),'yyyy-MM-dd hh:mm:ss') + new Date(params.row.updateTime).toLocaleString() ) } }, @@ -248,16 +248,17 @@ export default { }, onModalOk(){ - this.$refs.errorCodeForm.formModel.submit((formData)=>{ + this.$refs.errorCodeForm.formModel.submit(async (formData)=>{ this.modalLoading = true if(this.modalAddMode=='add') { - add(formData).then((data)=>{ + await add(formData).then((data)=>{ window.console.log(data) if(data.result) { this.$Message.success({ duration: 3, content: this.$t('message.linkis.basedataManagement.modal.modalAddSuccess') }) + this.load(); }else{ this.$Message.success({ duration: 3, @@ -266,7 +267,7 @@ export default { } }) }else { - edit(formData).then((data)=>{ + await edit(formData).then((data)=>{ window.console.log(data) if(data.result) { this.$Message.success({ diff --git a/linkis-web/src/common/i18n/en.json b/linkis-web/src/common/i18n/en.json index bd0a869d14..db26f22182 100644 --- a/linkis-web/src/common/i18n/en.json +++ b/linkis-web/src/common/i18n/en.json @@ -309,6 +309,7 @@ "resultGroupLineFilter": "Result Group Line Filter", "resultGroup": "Result Group", "all": "all", + "autoFormat": "Auto Format", "downloadMode": "Download Mode :", "deepAnalysis": "Deep Analysis", "success": { diff --git a/linkis-web/src/common/i18n/zh.json b/linkis-web/src/common/i18n/zh.json index 9491068eeb..a38fa3deb8 100644 --- a/linkis-web/src/common/i18n/zh.json +++ b/linkis-web/src/common/i18n/zh.json @@ -315,6 +315,7 @@ "resultGroupLineFilter": "对结果集的列进行筛选", "resultGroup": "结果集", "all": "是否全量", + "autoFormat": "自动格式化", "downloadMode": "下载方式：", "deepAnalysis": "分 析 进 阶", "success": { diff --git a/linkis-web/src/components/consoleComponent/resultSetList.vue b/linkis-web/src/components/consoleComponent/resultSetList.vue index 22f4e24e74..80c8a99d73 100644 --- a/linkis-web/src/components/consoleComponent/resultSetList.vue +++ b/linkis-web/src/components/consoleComponent/resultSetList.vue @@ -45,7 +45,7 @@ v-for="(item, index) in list" :class="{current: current-0 === index}" :data-index="index" - :key="item.path">{{$t('message.common.resultList')}}{{ index+1 }} + :key="item.path">{{$t('message.common.resultList')}}{{ +item.name.split('.')[0].substring(1) + 1 }}
@@ -73,7 +73,9 @@ export default { }, data() { return { - resultList: this.list, + resultList: this.list.sort((a, b) => { + return +(a.name.split('.')[0].substring(1)) - +(b.name.split('.')[0].substring(1)) + }), show: false, }; }, diff --git a/linkis-web/src/components/consoleComponent/toolbar.vue b/linkis-web/src/components/consoleComponent/toolbar.vue index ccbc4f738e..20bcd8e67b 100644 --- a/linkis-web/src/components/consoleComponent/toolbar.vue +++ b/linkis-web/src/components/consoleComponent/toolbar.vue @@ -150,6 +150,9 @@ {{$t('message.common.toolbar.all')}} + + {{$t('message.common.toolbar.autoFormat')}} + @@ -267,6 +270,7 @@ export default { isIconLabelShow: true, iconSize: 14, allDownload: false, // whether to download all result sets(是否下载全部结果集) + autoFormat: false, // whether to format result sets resultsShowType: '2', separators: [ { key: ',', label: this.$t('message.common.separator.comma'), value: '1', span: 4, offset: 0}, @@ -369,6 +373,9 @@ export default { let separator = encodeURIComponent(separatorItem.key || ''); url += `&csvSeparator=${separator}` } + if(this.isAll) { + url += `&autoFormat=${this.autoFormat}` + } // Before downloading, use the heartbeat interface to confirm whether to log in(下载之前条用心跳接口确认是否登录) await api.fetch('/user/heartbeat', 'get'); const link = document.createElement('a'); From 07f17c517f9daab625b63a026adecc1bdcaf9067 Mon Sep 17 00:00:00 2001 From: ahaoyao <129247228+ahaoyao@users.noreply.github.com> Date: Tue, 11 Apr 2023 14:44:12 +0800 Subject: [PATCH 057/261] feat:Add single test to ByteTimeUtils and fix code that triggers null pointer exception (#126) --- .../linkis/common/utils/ByteTimeUtils.java | 1 - .../common/utils/ByteTimeUtilsTest.java | 168 +++++++++++++++--- 2 files changed, 146 insertions(+), 23 deletions(-) diff --git a/linkis-commons/linkis-common/src/main/java/org/apache/linkis/common/utils/ByteTimeUtils.java b/linkis-commons/linkis-common/src/main/java/org/apache/linkis/common/utils/ByteTimeUtils.java index d23f4a0867..0fcb70d572 100644 --- a/linkis-commons/linkis-common/src/main/java/org/apache/linkis/common/utils/ByteTimeUtils.java +++ b/linkis-commons/linkis-common/src/main/java/org/apache/linkis/common/utils/ByteTimeUtils.java @@ -213,7 +213,6 @@ private static long parseByteString(String str, ByteUnit unit) { } else { throw new NumberFormatException("Failed to parse byte string: " + str); } - suffix = suffix.toLowerCase(); // Check for invalid suffixes if (suffix != null && !byteSuffixes.containsKey(suffix)) { throw new NumberFormatException("Invalid suffix: \"" + suffix + "\""); diff --git a/linkis-commons/linkis-common/src/test/java/org/apache/linkis/common/utils/ByteTimeUtilsTest.java b/linkis-commons/linkis-common/src/test/java/org/apache/linkis/common/utils/ByteTimeUtilsTest.java index 0d520cc05e..f548d89d46 100644 --- a/linkis-commons/linkis-common/src/test/java/org/apache/linkis/common/utils/ByteTimeUtilsTest.java +++ b/linkis-commons/linkis-common/src/test/java/org/apache/linkis/common/utils/ByteTimeUtilsTest.java @@ -17,35 +17,159 @@ package org.apache.linkis.common.utils; -import static org.junit.jupiter.api.Assertions.*; +import java.util.function.Function; -import org.apache.linkis.common.exception.LinkisSecurityException; +import com.google.common.collect.ImmutableMap; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; + class ByteTimeUtilsTest { - @Test - void byteStringAsBytes() { - } + private static final ImmutableMap> opFunction = + ImmutableMap.>builder() + .put("byteStringAsBytes", tar -> ByteTimeUtils.byteStringAsBytes(tar)) + .put("byteStringAsKb", tar -> ByteTimeUtils.byteStringAsKb(tar)) + .put("byteStringAsMb", tar -> ByteTimeUtils.byteStringAsMb(tar)) + .put("byteStringAsGb", tar -> ByteTimeUtils.byteStringAsGb(tar)) + .build(); + + private static final ImmutableMap convertToByte = + ImmutableMap.builder() + .put("1", 1l) + .put("1b", 1l) + .put("1B", 1l) + .put("1k", 1024l) + .put("1K", 1024l) + .put("1kb", 1024l) + .put("1Kb", 1024l) + .put("1kB", 1024l) + .put("1KB", 1024l) + .put("1m", 1024l * 1024l) + .put("1M", 1024l * 1024l) + .put("1mb", 1024l * 1024l) + .put("1Mb", 1024l * 1024l) + .put("1mB", 1024l * 1024l) + .put("1MB", 1024l * 1024l) + .put("1g", 1024l * 1024l * 1024l) + .put("1G", 1024l * 1024l * 1024l) + .put("1gb", 1024l * 1024l * 1024l) + .put("1gB", 1024l * 1024l * 1024l) + .put("1Gb", 1024l * 1024l * 1024l) + .put("1GB", 1024l * 1024l * 1024l) + .put("1t", 1024l * 1024l * 1024l * 1024l) + .put("1T", 1024l * 1024l * 1024l * 1024l) + .put("1tb", 1024l * 1024l * 1024l * 1024l) + .put("1Tb", 1024l * 1024l * 1024l * 1024l) + .put("1tB", 1024l * 1024l * 1024l * 1024l) + .put("1TB", 1024l * 1024l * 1024l * 1024l) + .put("1p", 1024l * 1024l * 1024l * 1024l * 1024l) + .put("1P", 1024l * 1024l * 1024l * 1024l * 1024l) + .put("1pb", 1024l * 1024l * 1024l * 1024l * 1024l) + .put("1Pb", 1024l * 1024l * 1024l * 1024l * 1024l) + .put("1pB", 1024l * 1024l * 1024l * 1024l * 1024l) + .put("1PB", 1024l * 1024l * 1024l * 1024l * 1024l) + .build(); + + private static final ImmutableMap convertToKB = + ImmutableMap.builder() + .put("1", 1l) + .put("1024b", 1l) + .put("1024B", 1l) + .put("1k", 1l) + .put("1K", 1l) + .put("1kb", 1l) + .put("1Kb", 1l) + .put("1kB", 1l) + .put("1KB", 1l) + .put("1m", 1024l) + .put("1M", 1024l) + .put("1mb", 1024l) + .put("1Mb", 1024l) + .put("1mB", 1024l) + .put("1MB", 1024l) + .put("1g", 1024l * 1024l) + .put("1G", 1024l * 1024l) + .put("1gb", 1024l * 1024l) + .put("1gB", 1024l * 1024l) + .put("1Gb", 1024l * 1024l) + .put("1GB", 1024l * 1024l) + .build(); + + private static final ImmutableMap convertToMB = + ImmutableMap.builder() + .put("1", 1l) + .put("1024k", 1l) + .put("1024K", 1l) + .put("1024kb", 1l) + .put("1024Kb", 1l) + .put("1024kB", 1l) + .put("1024KB", 1l) + .put("1m", 1l) + .put("1M", 1l) + .put("1mb", 1l) + .put("1Mb", 1l) + .put("1mB", 1l) + .put("1MB", 1l) + .put("1g", 1024l) + .put("1G", 1024l) + .put("1gb", 1024l) + .put("1gB", 1024l) + .put("1Gb", 1024l) + .put("1GB", 1024l) + .build(); - @Test - void byteStringAsKb() { - } + private static final ImmutableMap convertToGB = + ImmutableMap.builder() + .put("1", 1l) + .put("1024m", 1l) + .put("1024M", 1l) + .put("1024mb", 1l) + .put("1024Mb", 1l) + .put("1024mB", 1l) + .put("1024MB", 1l) + .put("1g", 1l) + .put("1G", 1l) + .put("1gb", 1l) + .put("1gB", 1l) + .put("1Gb", 1l) + .put("1GB", 1l) + .put("1t", 1024l) + .put("1T", 1024l) + .put("1tb", 1024l) + .put("1Tb", 1024l) + .put("1tB", 1024l) + .put("1TB", 1024l) + .build(); - @Test - void byteStringAsMb() { - } + @Test + void byteStringAsBytes() { + convertToByte.forEach( + (k, v) -> Assertions.assertEquals(opFunction.get("byteStringAsBytes").apply(k), v)); + Assertions.assertThrows( + IllegalArgumentException.class, () -> opFunction.get("byteStringAsBytes").apply("1A")); + } - @Test - void byteStringAsGb() { - Long res = ByteTimeUtils.byteStringAsGb("1G"); - Assertions.assertEquals(res,1); + @Test + void byteStringAsKb() { + convertToKB.forEach( + (k, v) -> Assertions.assertEquals(opFunction.get("byteStringAsKb").apply(k), v)); + Assertions.assertThrows( + IllegalArgumentException.class, () -> opFunction.get("byteStringAsKb").apply("1a")); + } - Assertions.assertThrows( - NullPointerException.class, - () -> { - ByteTimeUtils.byteStringAsGb("512"); - }); + @Test + void byteStringAsMb() { + convertToMB.forEach( + (k, v) -> Assertions.assertEquals(opFunction.get("byteStringAsMb").apply(k), v)); + Assertions.assertThrows( + IllegalArgumentException.class, () -> opFunction.get("byteStringAsMb").apply("1c")); + } - } -} \ No newline at end of file + @Test + void byteStringAsGb() { + convertToGB.forEach( + (k, v) -> Assertions.assertEquals(opFunction.get("byteStringAsGb").apply(k), v)); + Assertions.assertThrows( + IllegalArgumentException.class, () -> opFunction.get("byteStringAsGb").apply("1C")); + } +} From c47f786d03326af809092f9880ec2dd2ffa204e9 Mon Sep 17 00:00:00 2001 From: ahaoyao <129247228+ahaoyao@users.noreply.github.com> Date: Tue, 11 Apr 2023 14:45:50 +0800 Subject: [PATCH 058/261] =?UTF-8?q?fix=EF=BC=9AFix=20spelling=20errors=20i?= =?UTF-8?q?n=20words=20(#129)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../am/service/engine/DefaultEngineCreateService.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/service/engine/DefaultEngineCreateService.scala b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/service/engine/DefaultEngineCreateService.scala index ca73d3fb3f..65e11f357c 100644 --- a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/service/engine/DefaultEngineCreateService.scala +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/service/engine/DefaultEngineCreateService.scala @@ -308,8 +308,8 @@ class DefaultEngineCreateService case AvailableResource(ticketId) => (ticketId, resource) case NotEnoughResource(reason) => - logger.warn(s"not engough resource: $reason") - throw new LinkisRetryException(AMConstant.EM_ERROR_CODE, s"not engough resource: : $reason") + logger.warn(s"not enough resource: $reason") + throw new LinkisRetryException(AMConstant.EM_ERROR_CODE, s"not enough resource: : $reason") } } From 0d57c9bb80b28a1e18957a548739caaa493a8759 Mon Sep 17 00:00:00 2001 From: ahaoyao <129247228+ahaoyao@users.noreply.github.com> Date: Tue, 11 Apr 2023 14:58:47 +0800 Subject: [PATCH 059/261] =?UTF-8?q?feat=EF=BC=9Aadd=20ldap-userNameFormat?= =?UTF-8?q?=20example=20value=20(#128)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- linkis-dist/package/conf/linkis-mg-gateway.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/linkis-dist/package/conf/linkis-mg-gateway.properties b/linkis-dist/package/conf/linkis-mg-gateway.properties index 84be3d897d..6a4f1d94a6 100644 --- a/linkis-dist/package/conf/linkis-mg-gateway.properties +++ b/linkis-dist/package/conf/linkis-mg-gateway.properties @@ -28,7 +28,7 @@ wds.linkis.login_encrypt.enable=false ##LDAP wds.linkis.ldap.proxy.url= wds.linkis.ldap.proxy.baseDN= -wds.linkis.ldap.proxy.userNameFormat= +#wds.linkis.ldap.proxy.userNameFormat=cn=%s@xxx.com,OU=xxx,DC=xxx,DC=com wds.linkis.admin.user=hadoop #wds.linkis.admin.password= ##Spring From 4b266bf2725792dbd4e37ca497e4cc2d3f3fa379 Mon Sep 17 00:00:00 2001 From: casionone Date: Tue, 11 Apr 2023 18:29:34 +0800 Subject: [PATCH 060/261] Revert "Fix custom variable settings (#107)" This reverts commit dd10fbcc --- .../script/writer/StorageScriptFsWriter.scala | 4 +- .../linkis/storage/source/FileSplit.scala | 6 +- .../writer/StorageScriptFsWriterTest.java | 87 ------------------- .../src/test/resources/scritpis-test2.sql | 9 -- pom.xml | 2 +- 5 files changed, 9 insertions(+), 99 deletions(-) delete mode 100644 linkis-commons/linkis-storage/src/test/resources/scritpis-test2.sql diff --git a/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/script/writer/StorageScriptFsWriter.scala b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/script/writer/StorageScriptFsWriter.scala index f8d4518c1d..cdb9186da4 100644 --- a/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/script/writer/StorageScriptFsWriter.scala +++ b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/script/writer/StorageScriptFsWriter.scala @@ -51,7 +51,9 @@ class StorageScriptFsWriter( .map(compaction.compact) .foreach(metadataLine.add) // add annotition symbol - metadataLine.add(compaction.getAnnotationSymbol()) + if (metadataLine.size() > 0) { + metadataLine.add(compaction.getAnnotationSymbol()) + } if (outputStream != null) { IOUtils.writeLines(metadataLine, "\n", outputStream, charset) } else { diff --git a/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/source/FileSplit.scala b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/source/FileSplit.scala index 3839800f12..467fbca335 100644 --- a/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/source/FileSplit.scala +++ b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/source/FileSplit.scala @@ -80,7 +80,11 @@ class FileSplit( var needRemoveFlag = false if (hasRemovedFlag == false && fsReader.isInstanceOf[StorageScriptFsReader]) { val parser = fsReader.asInstanceOf[StorageScriptFsReader].getScriptParser() - if (parser != null && parser.getAnnotationSymbol().equals(record.toString)) { + val meta = metaData.asInstanceOf[ScriptMetaData].getMetaData + if ( + meta != null && meta.length > 0 + && parser != null && parser.getAnnotationSymbol().equals(record.toString) + ) { needRemoveFlag = true hasRemovedFlag = true } diff --git a/linkis-commons/linkis-storage/src/test/java/org/apache/linkis/storage/script/writer/StorageScriptFsWriterTest.java b/linkis-commons/linkis-storage/src/test/java/org/apache/linkis/storage/script/writer/StorageScriptFsWriterTest.java index 523402c2ef..60a593665c 100644 --- a/linkis-commons/linkis-storage/src/test/java/org/apache/linkis/storage/script/writer/StorageScriptFsWriterTest.java +++ b/linkis-commons/linkis-storage/src/test/java/org/apache/linkis/storage/script/writer/StorageScriptFsWriterTest.java @@ -98,43 +98,6 @@ class StorageScriptFsWriterTest { String fileName = "229cf765-6839-4c82-829d-1907c2ccf668.sql"; - String scriptContent2 = - "" - + "select ${qq};\n" - + "--@set test=123\n" - + "select ${test};\n" - + "--@set qq=222\n" - + "select ${qq};\n" - + "--\n" - + "--\n" - + "select 1;"; - String metaData2 = ""; - - String resultMetaData2 = "{}"; - String resultString2 = - "" - + "--\n" - + "select ${qq};\n" - + "--@set test=123\n" - + "select ${test};\n" - + "--@set qq=222\n" - + "select ${qq};\n" - + "--\n" - + "--\n" - + "select 1;"; - - Map params2 = new HashMap<>(); - - { - try { - params = new ObjectMapper().readValue(metaData, HashMap.class); - } catch (JsonProcessingException e) { - e.printStackTrace(); - } - } - - String fileName2 = "229cf765-6839-4c82-829d-1907c2ccf669.sql"; - // return Message.ok().data("resourceId", resourceId).data("version", version); @Test @@ -164,34 +127,6 @@ void TestSave() { } } - @Test - void TestSave2() { - - ScriptFsWriter writer = - StorageScriptFsWriter.getScriptFsWriter(new FsPath(fileName2), "UTF-8", null); - Variable[] v = VariableParser.getVariables(params2); - List variableList = - Arrays.stream(v) - .filter(var -> !StringUtils.isEmpty(var.value())) - .collect(Collectors.toList()); - try { - - MetaData getMetaData = new ScriptMetaData(variableList.toArray(new Variable[0])); - writer.addMetaData(getMetaData); - writer.addRecord(new ScriptRecord(scriptContent2)); - InputStream inputStream = writer.getInputStream(); - - String text = - new BufferedReader(new InputStreamReader(inputStream, StandardCharsets.UTF_8)) - .lines() - .collect(Collectors.joining("\n")); - Assertions.assertEquals(text, resultString2); - - } catch (Exception e) { - e.printStackTrace(); - } - } - @Test void TestOpen() throws FileNotFoundException { @@ -213,26 +148,4 @@ void TestOpen() throws FileNotFoundException { Assertions.assertEquals(metadataRes, resultMetaData); } - - @Test - void TestOpen2() throws FileNotFoundException { - - // /api/rest_j/v1/filesystem/openScriptFromBML?fileName=229cf765-6839-4c82-829d-1907c2ccf668.sql&resourceId=c9755cad-619b-4c1c-9204-cc4bb9836194&version=v000008&creator=&projectName=test1122a1 - - String filePath = this.getClass().getResource("/scritpis-test2.sql").getFile().toString(); - - File file = new File(filePath); - - InputStream inputStream = new FileInputStream(file); - - FileSource fileSource = FileSource$.MODULE$.create(new FsPath(fileName2), inputStream); - Pair> collect = fileSource.collect()[0]; - - String scriptRes = collect.getSecond().get(0)[0]; - String metadataRes = new Gson().toJson(collect.getFirst()); - - Assertions.assertEquals(scriptContent2 + "\n", scriptRes); - - Assertions.assertEquals(metadataRes, resultMetaData2); - } } diff --git a/linkis-commons/linkis-storage/src/test/resources/scritpis-test2.sql b/linkis-commons/linkis-storage/src/test/resources/scritpis-test2.sql deleted file mode 100644 index 8f4a6defbe..0000000000 --- a/linkis-commons/linkis-storage/src/test/resources/scritpis-test2.sql +++ /dev/null @@ -1,9 +0,0 @@ --- -select ${qq}; ---@set test=123 -select ${test}; ---@set qq=222 -select ${qq}; --- --- -select 1; \ No newline at end of file diff --git a/pom.xml b/pom.xml index fe5de420b1..0eaa57f1f5 100644 --- a/pom.xml +++ b/pom.xml @@ -1184,7 +1184,7 @@ compiler/** **/generated/** **/scritpis-test.sql - **/scritpis-test2.sql + From 3bf1d8ef0659477971f73f18fbfa8329ee3d4756 Mon Sep 17 00:00:00 2001 From: huangKai-2323 <62878639+huangKai-2323@users.noreply.github.com> Date: Thu, 13 Apr 2023 19:30:00 +0800 Subject: [PATCH 061/261] fix JobHistoryQueryServiceImpl cache bug (#133) --- .../jobhistory/service/impl/JobHistoryQueryServiceImpl.scala | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/linkis-public-enhancements/linkis-jobhistory/src/main/scala/org/apache/linkis/jobhistory/service/impl/JobHistoryQueryServiceImpl.scala b/linkis-public-enhancements/linkis-jobhistory/src/main/scala/org/apache/linkis/jobhistory/service/impl/JobHistoryQueryServiceImpl.scala index 6778276d3a..e0faea535c 100644 --- a/linkis-public-enhancements/linkis-jobhistory/src/main/scala/org/apache/linkis/jobhistory/service/impl/JobHistoryQueryServiceImpl.scala +++ b/linkis-public-enhancements/linkis-jobhistory/src/main/scala/org/apache/linkis/jobhistory/service/impl/JobHistoryQueryServiceImpl.scala @@ -353,10 +353,9 @@ class JobHistoryQueryServiceImpl extends JobHistoryQueryService with Logging { startJobId: lang.Long ): Integer = { val cacheKey = - if (StringUtils.isNoneBlank(username, creator, engineType)) "" - else { + if (StringUtils.isNoneBlank(username, creator, engineType)) { s"${username}_${creator}_${engineType}" - } + } else "" if (StringUtils.isBlank(cacheKey)) { getCountUndoneTasks(username, creator, sDate, eDate, engineType, startJobId) } else { From 5ce48b29f56455440c8e37876889f2c48a179e19 Mon Sep 17 00:00:00 2001 From: huangKai-2323 <62878639+huangKai-2323@users.noreply.github.com> Date: Tue, 18 Apr 2023 10:01:03 +0800 Subject: [PATCH 062/261] update spark datatype --- .../linkis/storage/domain/DataType.scala | 31 ++++++++++++++++++- .../linkis/storage/domain/DataTypeTest.scala | 8 +++++ .../spark/executor/SQLSession.scala | 7 ++++- 3 files changed, 44 insertions(+), 2 deletions(-) diff --git a/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/domain/DataType.scala b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/domain/DataType.scala index 55c82abd38..365c732454 100644 --- a/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/domain/DataType.scala +++ b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/domain/DataType.scala @@ -79,7 +79,16 @@ object DataType extends Logging { case IntType => if (isNumberNull(value)) null else value.toInt case LongType | BigIntType => if (isNumberNull(value)) null else value.toLong case FloatType => if (isNumberNull(value)) null else value.toFloat - case DoubleType => if (isNumberNull(value)) null else value.toDouble + case DoubleType => + logger.info("--------------------- value : {}", value) + logger.info("--------------------- result : {}", value.equals("NaN")) + if (isNumberNull(value)) { + null + } else if (value.equals("NaN")) { + "NaN" + } else { + value.toDouble + } case DecimalType => if (isNumberNull(value)) null else new JavaBigDecimal(value) case DateType => if (isNumberNull(value)) null else Date.valueOf(value) case TimestampType => @@ -117,25 +126,45 @@ abstract class DataType(val typeName: String, val javaSQLType: Int) { } case object NullType extends DataType("void", 0) + case object StringType extends DataType("string", 12) + case object BooleanType extends DataType("boolean", 16) + case object TinyIntType extends DataType("tinyint", -6) + case object ShortIntType extends DataType("short", 5) + case object IntType extends DataType("int", 4) + case object LongType extends DataType("long", -5) + case object BigIntType extends DataType("bigint", -5) + case object FloatType extends DataType("float", 6) + case object DoubleType extends DataType("double", 8) + case object CharType extends DataType("char", 1) + case object VarcharType extends DataType("varchar", 12) + case object DateType extends DataType("date", 91) + case object TimestampType extends DataType("timestamp", 93) + case object BinaryType extends DataType("binary", -2) + case object DecimalType extends DataType("decimal", 3) + case object ArrayType extends DataType("array", 2003) + case object MapType extends DataType("map", 2000) + case object ListType extends DataType("list", 2001) + case object StructType extends DataType("struct", 2002) + case object BigDecimalType extends DataType("bigdecimal", 3) case class Column(columnName: String, dataType: DataType, comment: String) { diff --git a/linkis-commons/linkis-storage/src/test/scala/org/apache/linkis/storage/domain/DataTypeTest.scala b/linkis-commons/linkis-storage/src/test/scala/org/apache/linkis/storage/domain/DataTypeTest.scala index 3bfc35b17c..81ea6b52bb 100644 --- a/linkis-commons/linkis-storage/src/test/scala/org/apache/linkis/storage/domain/DataTypeTest.scala +++ b/linkis-commons/linkis-storage/src/test/scala/org/apache/linkis/storage/domain/DataTypeTest.scala @@ -60,4 +60,12 @@ class DataTypeTest { } + @Test + @DisplayName("toValueTest") + def toValueTest(): Unit = { + val dateType = DataType.toDataType("double") + val str = DataType.toValue(dateType, "NaN") + Assertions.assertNotNull(str) + } + } diff --git a/linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/executor/SQLSession.scala b/linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/executor/SQLSession.scala index 3291304868..9671ab6922 100644 --- a/linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/executor/SQLSession.scala +++ b/linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/executor/SQLSession.scala @@ -214,7 +214,12 @@ object SQLSession extends Logging { .mkString("{", ",", "}") case (str: String, StringType) => str.replaceAll("\n|\t", " ") - case (double: Double, DoubleType) => nf.format(double) + case (double: Double, DoubleType) => + if (double.isNaN) { + "NaN" + } else { + nf.format(double) + } case (decimal: java.math.BigDecimal, DecimalType()) => formatDecimal(decimal) case (other: Any, tpe) => other.toString case _ => null From a488132a7ed562f12b6f9a8d74a95bd3c3e9e05a Mon Sep 17 00:00:00 2001 From: HmhWz <934178858@qq.com> Date: Thu, 20 Apr 2023 21:00:04 +0800 Subject: [PATCH 063/261] Compatible with trino version 407. --- .../engineplugin/trino/conf/TrinoConfiguration.scala | 2 ++ .../trino/executor/TrinoEngineConnExecutor.scala | 11 +++++++++-- .../linkis/engineplugin/trino/utils/TrinoCode.scala | 12 ++++++++++++ 3 files changed, 23 insertions(+), 2 deletions(-) diff --git a/linkis-engineconn-plugins/trino/src/main/scala/org/apache/linkis/engineplugin/trino/conf/TrinoConfiguration.scala b/linkis-engineconn-plugins/trino/src/main/scala/org/apache/linkis/engineplugin/trino/conf/TrinoConfiguration.scala index e5c6f5186a..cfeb943f01 100644 --- a/linkis-engineconn-plugins/trino/src/main/scala/org/apache/linkis/engineplugin/trino/conf/TrinoConfiguration.scala +++ b/linkis-engineconn-plugins/trino/src/main/scala/org/apache/linkis/engineplugin/trino/conf/TrinoConfiguration.scala @@ -57,6 +57,8 @@ object TrinoConfiguration { val TRINO_FORBID_GRANT = CommonVars[Boolean]("linkis.trino.forbid.grant", true) + val TRINO_FORBID_FETCHFIRST = CommonVars[Boolean]("linkis.trino.forbid.fetchfirst", true) + val TRINO_FORBID_MODIFY_SCHEMA = CommonVars[Boolean]("linkis.trino.forbid.modifySchema", true) diff --git a/linkis-engineconn-plugins/trino/src/main/scala/org/apache/linkis/engineplugin/trino/executor/TrinoEngineConnExecutor.scala b/linkis-engineconn-plugins/trino/src/main/scala/org/apache/linkis/engineplugin/trino/executor/TrinoEngineConnExecutor.scala index a3a0d0d900..82638d03bc 100644 --- a/linkis-engineconn-plugins/trino/src/main/scala/org/apache/linkis/engineplugin/trino/executor/TrinoEngineConnExecutor.scala +++ b/linkis-engineconn-plugins/trino/src/main/scala/org/apache/linkis/engineplugin/trino/executor/TrinoEngineConnExecutor.scala @@ -400,14 +400,21 @@ class TrinoEngineConnExecutor(override val outputPrintLimit: Int, val id: Int) } else { results = statement.finalStatusInfo() } + // v407中create table时columns返回为null if (results.getColumns == null) { - throw new RuntimeException("trino columns is null.") + // throw new RuntimeException("trino columns is null.") + logger.info(s"results columns is null for task: $taskId") + return } + val columns = results.getColumns.asScala .map(column => Column(column.getName, column.getType, "")) .toArray[Column] + // 兼容结果集中colums为空集合的情况，如use db, drop table if exists + if (columns.length != 0) { + resultSetWriter.addMetaData(new TableMetaData(columns)) + } columnCount = columns.length - resultSetWriter.addMetaData(new TableMetaData(columns)) while (statement.isRunning) { val data = statement.currentData().getData if (data != null) for (row <- data.asScala) { diff --git a/linkis-engineconn-plugins/trino/src/main/scala/org/apache/linkis/engineplugin/trino/utils/TrinoCode.scala b/linkis-engineconn-plugins/trino/src/main/scala/org/apache/linkis/engineplugin/trino/utils/TrinoCode.scala index 6329969259..42c9bcf461 100644 --- a/linkis-engineconn-plugins/trino/src/main/scala/org/apache/linkis/engineplugin/trino/utils/TrinoCode.scala +++ b/linkis-engineconn-plugins/trino/src/main/scala/org/apache/linkis/engineplugin/trino/utils/TrinoCode.scala @@ -19,6 +19,7 @@ package org.apache.linkis.engineplugin.trino.utils import org.apache.linkis.engineplugin.trino.conf.TrinoConfiguration import org.apache.linkis.engineplugin.trino.exception.{ + TrinoClientException, TrinoGrantmaException, TrinoModifySchemaException } @@ -39,6 +40,14 @@ object TrinoCode { .contains("alter schema") } + private def hasFetchFirst(code: String): Boolean = { + val trimmedCode = code + .replaceAll("\n", SPACE) + .replaceAll("\\s+", SPACE) + .toLowerCase() + trimmedCode.matches(".*fetch first (\\d+ )?row only.*") + } + def checkCode(code: String): Unit = { if ( TrinoConfiguration.TRINO_FORBID_MODIFY_SCHEMA.getValue && TrinoCode.willModifySchema(code) @@ -48,6 +57,9 @@ object TrinoCode { if (TrinoConfiguration.TRINO_FORBID_GRANT.getValue && TrinoCode.willGrant(code)) { throw TrinoGrantmaException("Grant schema or table is not allowed") } + if (TrinoConfiguration.TRINO_FORBID_FETCHFIRST.getValue && TrinoCode.hasFetchFirst(code)) { + throw TrinoClientException("Fetch first row only is not allowed, please use limit n.") + } } } From 5ff9f98dc5ba0660e18fb2d17d19f053560ad9e3 Mon Sep 17 00:00:00 2001 From: mhHao <934178858@qq.com> Date: Fri, 21 Apr 2023 23:11:27 +0800 Subject: [PATCH 064/261] add dss gateway support parser. (#135) --- .../gateway/dss/DSSGatewayConfiguration.scala | 42 +++ .../linkis/gateway/dss/DSSGatewayParser.scala | 290 ++++++++++++++++++ .../gateway/dss/DSSRouteLabelParser.scala | 77 +++++ 3 files changed, 409 insertions(+) create mode 100644 linkis-spring-cloud-services/linkis-service-gateway/linkis-gateway-server-support/src/main/scala/org/apache/linkis/gateway/dss/DSSGatewayConfiguration.scala create mode 100644 linkis-spring-cloud-services/linkis-service-gateway/linkis-gateway-server-support/src/main/scala/org/apache/linkis/gateway/dss/DSSGatewayParser.scala create mode 100644 linkis-spring-cloud-services/linkis-service-gateway/linkis-gateway-server-support/src/main/scala/org/apache/linkis/gateway/dss/DSSRouteLabelParser.scala diff --git a/linkis-spring-cloud-services/linkis-service-gateway/linkis-gateway-server-support/src/main/scala/org/apache/linkis/gateway/dss/DSSGatewayConfiguration.scala b/linkis-spring-cloud-services/linkis-service-gateway/linkis-gateway-server-support/src/main/scala/org/apache/linkis/gateway/dss/DSSGatewayConfiguration.scala new file mode 100644 index 0000000000..a649e9c182 --- /dev/null +++ b/linkis-spring-cloud-services/linkis-service-gateway/linkis-gateway-server-support/src/main/scala/org/apache/linkis/gateway/dss/DSSGatewayConfiguration.scala @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.gateway.dss + +import org.apache.linkis.common.conf.CommonVars + +object DSSGatewayConfiguration { + val DSS_SPRING_NAME: CommonVars[String] = CommonVars("wds.linkis.dss.name", "dss-server") + + val DSS_URL_LABEL_PREFIX: CommonVars[String] = + CommonVars("wds.dss.gateway.url.prefix.name", "labels") + + val DSS_URL_ROUTE_LABEL_PREFIX: CommonVars[String] = + CommonVars("wds.dss.gateway.url.prefix.name", "labelsRoute") + + val DSS_URL_APPCONNS: CommonVars[String] = CommonVars("wds.dss.gateway.url.appconns", "visualis") + + val DSS_APPS_SERVER_OTHER_PREFIX: CommonVars[String] = + CommonVars("wds.dss.gateway.apps.server.other.prefix", "scriptis,apiservice,datapipe,guide") + + val DSS_APPS_SERVER_DISTINCT_NAME: CommonVars[String] = + CommonVars("wds.dss.gateway.apps.server.distinct.name", "apps") + + val DSS_APPS_SERVER_ISMERGE: CommonVars[Boolean] = + CommonVars("wds.dss.gateway.apps.server.ismerge", true) + +} diff --git a/linkis-spring-cloud-services/linkis-service-gateway/linkis-gateway-server-support/src/main/scala/org/apache/linkis/gateway/dss/DSSGatewayParser.scala b/linkis-spring-cloud-services/linkis-service-gateway/linkis-gateway-server-support/src/main/scala/org/apache/linkis/gateway/dss/DSSGatewayParser.scala new file mode 100644 index 0000000000..cca49edcdc --- /dev/null +++ b/linkis-spring-cloud-services/linkis-service-gateway/linkis-gateway-server-support/src/main/scala/org/apache/linkis/gateway/dss/DSSGatewayParser.scala @@ -0,0 +1,290 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.gateway.dss + +import org.apache.linkis.common.ServiceInstance +import org.apache.linkis.gateway.exception.TooManyServiceException +import org.apache.linkis.gateway.http.GatewayContext +import org.apache.linkis.gateway.parser.AbstractGatewayParser +import org.apache.linkis.gateway.springcloud.SpringCloudGatewayConfiguration.{ + normalPath, + API_URL_PREFIX +} +import org.apache.linkis.manager.label.builder.factory.LabelBuilderFactoryContext +import org.apache.linkis.manager.label.entity.Label +import org.apache.linkis.manager.label.entity.route.RouteLabel +import org.apache.linkis.protocol.constants.TaskConstant +import org.apache.linkis.protocol.utils.ZuulEntranceUtils +import org.apache.linkis.rpc.sender.SpringCloudFeignConfigurationCache +import org.apache.linkis.server.BDPJettyServerHelper + +import org.springframework.stereotype.Component + +import java.util +import java.util.Locale + +import scala.collection.JavaConversions._ + +@Component +class DSSGatewayParser extends AbstractGatewayParser { + + val appConns = DSSGatewayConfiguration.DSS_URL_APPCONNS.getValue.split(",") + + override def shouldContainRequestBody(gatewayContext: GatewayContext): Boolean = { + var contentType = gatewayContext.getRequest.getHeaders.get("Content-Type") + if (null == contentType) { + contentType = gatewayContext.getRequest.getHeaders.get("content-type") + } + + if ( + contentType != null && contentType.nonEmpty + && contentType(0).contains("form-data") + ) { + logger.info("DSS gateway get request type is form-data") + return false + } + + gatewayContext.getRequest.getRequestURI match { + case DSSGatewayParser.DSS_URL_DEFAULT_REGEX(_, _) => true + case DSSGatewayParser.DSS_URL_REGEX(_, _, _) => true + case DSSGatewayParser.APPCONN_URL_DEFAULT_REGEX(_, appconn, _) + if appConns.contains(appconn) => + true + case _ => false + } + } + + override def parse(gatewayContext: GatewayContext): Unit = + gatewayContext.getRequest.getRequestURI match { + + case DSSGatewayParser.DSS_URL_FLOW_QUERY_PREFIX(version, execId, _) => + // must put it before DSS_URL_REGEX(_, _, _), because this match was included in DSS_URL_REGEX(_, _, _) + if (sendResponseWhenNotMatchVersion(gatewayContext, version)) return + val serviceInstances = ZuulEntranceUtils.parseServiceInstanceByExecID(execId) + gatewayContext.getGatewayRoute.setServiceInstance(serviceInstances(0)) + case DSSGatewayParser.DSS_URL_REGEX(version, firstName, secondName) => + if (sendResponseWhenNotMatchVersion(gatewayContext, version)) return + var tmpServerName = "dss-" + firstName + "-" + secondName + "-server" + tmpServerName = getServiceNameFromLabel(gatewayContext, tmpServerName) + // apiservice,datapipe,scriptis和guide服务合并到dss-apps-server，其中的接口需要转发到apps服务 + var tmpFirstName = firstName + if ( + DSSGatewayConfiguration.DSS_APPS_SERVER_ISMERGE.getValue && + DSSGatewayConfiguration.DSS_APPS_SERVER_OTHER_PREFIX.getValue + .split(",") + .contains(firstName) + ) { + tmpFirstName = + DSSGatewayConfiguration.DSS_APPS_SERVER_DISTINCT_NAME.getValue + "/" + firstName + } + val serviceName: Option[String] = + findCommonService("dss/" + tmpFirstName + "/" + secondName, tmpServerName) + if (serviceName.isDefined) { + gatewayContext.getGatewayRoute.setServiceInstance(ServiceInstance(serviceName.get, null)) + } else { + logger.info( + "Now set default serviceInstance name " + DSSGatewayConfiguration.DSS_SPRING_NAME.getValue + "," + gatewayContext.getRequest.getRequestURI + ) + gatewayContext.getGatewayRoute.setServiceInstance( + ServiceInstance(DSSGatewayConfiguration.DSS_SPRING_NAME.getValue, null) + ) + } + case DSSGatewayParser.DSS_URL_DEFAULT_REGEX(version, firstName) => + if (sendResponseWhenNotMatchVersion(gatewayContext, version)) return + var tmpServerName = "dss-" + firstName + "-server" + tmpServerName = getServiceNameFromLabel(gatewayContext, tmpServerName) + // apiservice,datapipe,scriptis和guide服务合并到dss-apps-server，其中的接口需要转发到apps服务 + var tmpFirstName = firstName + if ( + DSSGatewayConfiguration.DSS_APPS_SERVER_ISMERGE.getValue && + DSSGatewayConfiguration.DSS_APPS_SERVER_OTHER_PREFIX.getValue + .split(",") + .contains(firstName) + ) { + tmpFirstName = + DSSGatewayConfiguration.DSS_APPS_SERVER_DISTINCT_NAME.getValue + "/" + firstName + } + val serviceName: Option[String] = findCommonService("dss/" + tmpFirstName, tmpServerName) + if (serviceName.isDefined) { + gatewayContext.getGatewayRoute.setServiceInstance(ServiceInstance(serviceName.get, null)) + } else { + logger.info( + "Now set default serviceInstance name " + DSSGatewayConfiguration.DSS_SPRING_NAME.getValue + "," + gatewayContext.getRequest.getRequestURI + ) + gatewayContext.getGatewayRoute.setServiceInstance( + ServiceInstance(DSSGatewayConfiguration.DSS_SPRING_NAME.getValue, null) + ) + } + case DSSGatewayParser.APPCONN_URL_DEFAULT_REGEX(version, serverName, _) + if appConns.contains(serverName) => + if (sendResponseWhenNotMatchVersion(gatewayContext, version)) return + var tmpServerName = serverName + tmpServerName = getServiceNameFromLabel(gatewayContext, tmpServerName) + val serviceName: Option[String] = findCommonService(tmpServerName, tmpServerName) + if (serviceName.isDefined) { + gatewayContext.getGatewayRoute.setServiceInstance(ServiceInstance(serviceName.get, null)) + } else { + logger.info( + "Now set default serviceInstance name " + DSSGatewayConfiguration.DSS_SPRING_NAME.getValue + "," + gatewayContext.getRequest.getRequestURI + ) + gatewayContext.getGatewayRoute.setServiceInstance( + ServiceInstance(DSSGatewayConfiguration.DSS_SPRING_NAME.getValue, null) + ) + } + case _ => + } + + private def getServiceNameFromLabel( + gatewayContext: GatewayContext, + tmpServiceName: String + ): String = { + var requestUrlLabels = gatewayContext.getRequest.getQueryParams + .getOrDefault(DSSGatewayConfiguration.DSS_URL_LABEL_PREFIX.getValue, null) + if (requestUrlLabels == null) { + requestUrlLabels = gatewayContext.getRequest.getQueryParams + .getOrDefault(DSSGatewayConfiguration.DSS_URL_ROUTE_LABEL_PREFIX.getValue, null) + } + logger.info( + "Get ServiceName From Label and method is " + gatewayContext.getRequest.getMethod.toString + ",and urlLabels is " + requestUrlLabels + ) + val requestMethod = gatewayContext.getRequest.getMethod.toLowerCase(Locale.ROOT) + if ( + requestUrlLabels == null && (requestMethod + .equals("post") || requestMethod.equals("put") || requestMethod.equals("delete")) + ) { + val requestBody = Option(gatewayContext.getRequest.getRequestBody) + val routeLabelList = new util.ArrayList[RouteLabel]() + + requestBody match { + // todo form-data resolve + case Some(body) => + val labelBuilderFactory = LabelBuilderFactoryContext.getLabelBuilderFactory + val json = + BDPJettyServerHelper.gson.fromJson(body, classOf[java.util.Map[String, Object]]) + val labels: util.List[Label[_]] = json.get(TaskConstant.LABELS) match { + case map: util.Map[String, Object] => labelBuilderFactory.getLabels(map) + case map: util.Map[String, Any] => labelBuilderFactory.getLabels(map.asInstanceOf) + case _ => new util.ArrayList[Label[_]]() + } + labels + .filter(label => label.isInstanceOf[RouteLabel]) + .foreach(label => { + routeLabelList.add(label.asInstanceOf[RouteLabel]) + }) + + case _ => null + } + val labelNameList = routeLabelList.map(routeLabel => routeLabel.getStringValue).toList + if (labelNameList != null && labelNameList.size > 0) { + genServiceNameByDSSLabel(labelNameList, tmpServiceName) + } else if (null != requestUrlLabels) { + genServiceNameByDSSLabel(requestUrlLabels.toList, tmpServiceName) + } else tmpServiceName + + } else { + if (requestUrlLabels != null) { + genServiceNameByDSSLabel(requestUrlLabels.toList, tmpServiceName) + } else tmpServiceName + } + } + + private def genServiceNameByDSSLabel(labelList: List[String], tmpServiceName: String): String = { + var resultName = tmpServiceName + if (null != labelList && labelList.size > 0) { + val labelNameList = labelList(0).replace(" ", "").split(",").toList + if (labelNameList.size > 0) { + if (labelNameList.find(name => name.equalsIgnoreCase("dev")).isDefined) { + resultName = tmpServiceName + "-dev" + } else if (labelNameList.find(name => name.equalsIgnoreCase("prod")).isDefined) { + resultName = tmpServiceName + "-prod" + } else if (labelNameList.find(name => name.equalsIgnoreCase("test")).isDefined) { + resultName = tmpServiceName + "-test" + } else { + resultName = tmpServiceName + } + } + } + resultName + } + + private def findCommonService(parsedServiceId: String, tmpServerName: String) = findService( + parsedServiceId, + tmpServerName, + services => { + val errorMsg = new TooManyServiceException( + s"Cannot find a correct serviceId for parsedServiceId $parsedServiceId, service list is: " + services + ) + warn("", errorMsg) + throw errorMsg + } + ) + + protected def findService( + parsedServiceId: String, + tmpServerName: String, + tooManyDeal: List[String] => Option[String] + ): Option[String] = { + val findIt: (String => Boolean) => Option[String] = op => { + val services = + SpringCloudFeignConfigurationCache.getDiscoveryClient.getServices.filter(op).toList + if (services.length == 1) Some(services.head) + else if (services.length > 1) tooManyDeal(services) + else None + } + // 通过匹配到最多的url中的path进行路由,如/dss/framework/workspace/ 会匹配到 dss-framework-workspace-server 而不是 dss-server + // 如果产生了相等的情况，则按照短的service名字为准 比如/dss/getProject, + // 我们可能会匹配到dss-server以及 dss-framework-workspace-server,则选择短名称的dss-server + val findMostCorrect: (String => (String, Int)) => Option[String] = { op => + { + val serviceMap = + SpringCloudFeignConfigurationCache.getDiscoveryClient.getServices.map(op).toMap + var count = 0 + var retService: Option[String] = None + serviceMap.foreach { case (k, v) => + if (v > count) { + count = v + retService = Some(k) + } else if (retService.isDefined && v == count && k.length < retService.get.length) { + retService = Some(k) + } + } + retService + } + } + val lowerServiceId = parsedServiceId.toLowerCase() + val serverName = tmpServerName.toLowerCase() + // findIt(_.toLowerCase() == lowerServiceId).orElse(findIt(_.toLowerCase().contains(lowerServiceId))) + findIt(_.toLowerCase() == serverName).orElse(findMostCorrect(service => { + (service, lowerServiceId.split("/").count(word => service.contains(word))) + })) + } + +} + +object DSSGatewayParser { + val DSS_HEADER = normalPath(API_URL_PREFIX) + "rest_[a-zA-Z][a-zA-Z_0-9]*/(v\\d+)/dss/" + val DSS_URL_REGEX = (DSS_HEADER + "([^/]+)/" + "([^/]+)/.+").r + val DSS_URL_DEFAULT_REGEX = (DSS_HEADER + "([^/]+).+").r + + val APPCONN_HEADER = normalPath(API_URL_PREFIX) + "rest_[a-zA-Z][a-zA-Z_0-9]*/(v\\d+)/([^/]+)/" + val APPCONN_URL_DEFAULT_REGEX = (APPCONN_HEADER + "([^/]+).+").r + + val DSS_URL_FLOW_QUERY_PREFIX = + (DSS_HEADER + "flow/entrance/" + "([^/]+)/" + "(status|execution)").r + +} diff --git a/linkis-spring-cloud-services/linkis-service-gateway/linkis-gateway-server-support/src/main/scala/org/apache/linkis/gateway/dss/DSSRouteLabelParser.scala b/linkis-spring-cloud-services/linkis-service-gateway/linkis-gateway-server-support/src/main/scala/org/apache/linkis/gateway/dss/DSSRouteLabelParser.scala new file mode 100644 index 0000000000..2587c823c6 --- /dev/null +++ b/linkis-spring-cloud-services/linkis-service-gateway/linkis-gateway-server-support/src/main/scala/org/apache/linkis/gateway/dss/DSSRouteLabelParser.scala @@ -0,0 +1,77 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.gateway.dss + +import org.apache.linkis.gateway.http.GatewayContext +import org.apache.linkis.gateway.ujes.route.label.RouteLabelParser +import org.apache.linkis.manager.label.builder.factory.LabelBuilderFactoryContext +import org.apache.linkis.manager.label.entity.Label +import org.apache.linkis.manager.label.entity.route.RouteLabel +import org.apache.linkis.protocol.constants.TaskConstant +import org.apache.linkis.server.BDPJettyServerHelper + +import org.springframework.stereotype.Component + +import java.util + +import scala.collection.JavaConversions._ + +@Component +class DSSRouteLabelParser extends RouteLabelParser { + + override def parse(gatewayContext: GatewayContext): util.List[RouteLabel] = { + val routeLabelList = new util.ArrayList[RouteLabel]() + var requestLabels = gatewayContext.getRequest.getQueryParams + .getOrDefault(DSSGatewayConfiguration.DSS_URL_LABEL_PREFIX.getValue, null) + if (requestLabels == null) { + requestLabels = gatewayContext.getRequest.getQueryParams + .getOrDefault(DSSGatewayConfiguration.DSS_URL_ROUTE_LABEL_PREFIX.getValue, null) + } + if (null != requestLabels && requestLabels.size > 0) { + val labelNameList = requestLabels(0).replace(" ", "").split(",").toList + if (labelNameList.size > 0) labelNameList.foreach(labelName => { + val routeLabel = new RouteLabel + routeLabel.setRoutePath(labelName) +// routeLabelList.add(routeLabel) + }) + } + if (routeLabelList.isEmpty) { + val requestBody = Option(gatewayContext.getRequest.getRequestBody) + requestBody match { + // todo form-data resolve + case Some(body) => + if (body.contains("form-data")) {} else { +// val labelBuilderFactory = LabelBuilderFactoryContext.getLabelBuilderFactory +// val json = BDPJettyServerHelper.gson.fromJson(body, classOf[java.util.Map[String, Object]]) +// val labels: util.List[Label[_]] = json.get(TaskConstant.LABELS) match { +// case map: util.Map[String, Object] => labelBuilderFactory.getLabels(map) +// case map: util.Map[String, Any] => labelBuilderFactory.getLabels(map.asInstanceOf) +// case _ => new util.ArrayList[Label[_]]() +// } +// labels.filter(label => label.isInstanceOf[RouteLabel]).foreach(label => { +// routeLabelList.add(label.asInstanceOf[RouteLabel]) +// }) + } + case _ => null + } + } + + routeLabelList + } + +} From 13440672cc9d0a6afb63df69a7abd59242d03343 Mon Sep 17 00:00:00 2001 From: ahaoyao Date: Tue, 11 Apr 2023 11:23:41 +0800 Subject: [PATCH 065/261] Supports the conversion of positive and negative byte numbers to gb --- .../linkis/common/utils/ByteTimeUtils.java | 12 ++++++ .../linkis/manager/rm/utils/RMUtils.scala | 2 +- .../linkis/manager/rm/utils/RMUtilsTest.scala | 43 +++++++++++++++++++ 3 files changed, 56 insertions(+), 1 deletion(-) create mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/test/scala/org/apache/linkis/manager/rm/utils/RMUtilsTest.scala diff --git a/linkis-commons/linkis-common/src/main/java/org/apache/linkis/common/utils/ByteTimeUtils.java b/linkis-commons/linkis-common/src/main/java/org/apache/linkis/common/utils/ByteTimeUtils.java index 0fcb70d572..0ecb3dc2a5 100644 --- a/linkis-commons/linkis-common/src/main/java/org/apache/linkis/common/utils/ByteTimeUtils.java +++ b/linkis-commons/linkis-common/src/main/java/org/apache/linkis/common/utils/ByteTimeUtils.java @@ -296,6 +296,18 @@ public static long byteStringAsGb(String str) { return parseByteString(str, ByteUnit.GiB); } + /** + * Convert a passed byte string (e.g. -50b, -100k, or -250m) to gibibytes for internal use. + * + *

If no suffix is provided, the passed number is assumed to be in gibibytes. + */ + public static long negativeByteStringAsGb(String str) { + if (str.startsWith("-")) { + return Math.negateExact(parseByteString(str.substring(1), ByteUnit.GiB)); + } + return parseByteString(str, ByteUnit.GiB); + } + /** * Returns a byte array with the buffer's contents, trying to avoid copying the data if possible. */ diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/rm/utils/RMUtils.scala b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/rm/utils/RMUtils.scala index ca5f420693..2a35b7403d 100644 --- a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/rm/utils/RMUtils.scala +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/rm/utils/RMUtils.scala @@ -222,7 +222,7 @@ object RMUtils extends Logging { if (logger.isDebugEnabled()) { logger.debug(s"Will change ${resource.toString} from ${unitType} to GB") } - ByteTimeUtils.byteStringAsGb(resource.toString + "b").toString + "GB" + ByteTimeUtils.negativeByteStringAsGb(resource.toString + "b").toString + "GB" } { case e: Exception => logger.error(s"Cannot convert ${resource} to Gb, " + e.getMessage) resource.toString + unitType diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/test/scala/org/apache/linkis/manager/rm/utils/RMUtilsTest.scala b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/test/scala/org/apache/linkis/manager/rm/utils/RMUtilsTest.scala new file mode 100644 index 0000000000..b95cd7d4f6 --- /dev/null +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/test/scala/org/apache/linkis/manager/rm/utils/RMUtilsTest.scala @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.manager.rm.utils + +import org.junit.jupiter.api.Test + +class RMUtilsTest { + + @Test def getResourceInfoMsg(): Unit = { + val resourceType: String = "Memory" + val unitType: String = "bytes" + val requestResource: Any = "644245094400" // 600G + val availableResource: Any = "-2147483648" // -2G + val maxResource: Any = "20454781747200" // 19050G + val result = RMUtils.getResourceInfoMsg( + resourceType, + unitType, + requestResource, + availableResource, + maxResource + ) + assert( + " user Memory, requestResource : 600GB > availableResource : -2GB, maxResource : 19050GB." + .equals(result) + ) + } + +} From bbc70653e8b79c547e3b97d12f949dd9ee5dfdbe Mon Sep 17 00:00:00 2001 From: ahaoyao Date: Thu, 13 Apr 2023 17:59:06 +0800 Subject: [PATCH 066/261] Optimization of token exception scenario description --- ...inkisGwAuthenticationErrorCodeSummary.java | 10 +++++-- .../service/CachedTokenService.scala | 28 +++++++++++++------ 2 files changed, 26 insertions(+), 12 deletions(-) diff --git a/linkis-spring-cloud-services/linkis-service-gateway/linkis-gateway-authentication/src/main/java/org/apache/linkis/gateway/authentication/errorcode/LinkisGwAuthenticationErrorCodeSummary.java b/linkis-spring-cloud-services/linkis-service-gateway/linkis-gateway-authentication/src/main/java/org/apache/linkis/gateway/authentication/errorcode/LinkisGwAuthenticationErrorCodeSummary.java index 5abff15380..bfdba4a28e 100644 --- a/linkis-spring-cloud-services/linkis-service-gateway/linkis-gateway-authentication/src/main/java/org/apache/linkis/gateway/authentication/errorcode/LinkisGwAuthenticationErrorCodeSummary.java +++ b/linkis-spring-cloud-services/linkis-service-gateway/linkis-gateway-authentication/src/main/java/org/apache/linkis/gateway/authentication/errorcode/LinkisGwAuthenticationErrorCodeSummary.java @@ -20,12 +20,16 @@ import org.apache.linkis.common.errorcode.LinkisErrorCode; public enum LinkisGwAuthenticationErrorCodeSummary implements LinkisErrorCode { - TOKEN_IS_NULL(15205, "token is null(token 令牌为空)!"), - FAILED_TO_LOAD_TOKEN(15200, "Failed to load token from DB into cache(无法将 token 令牌从数据库加载到缓存中)!"), + FAILED_TO_LOAD_TOKEN( + 15200, + "Failed to load token:{0} from DB into cache(无法将 Token:{0} 令牌从数据库加载到缓存中),Caused by:{1}"), TOKEN_VALID_OR_STALE(15201, "Token is not valid or stale(token 令牌无效或已过期)!"), ILLEGAL_TOKENUSER(15202, "Illegal TokenUser for Token(Token非法用户)!"), ILLEGAL_HOST(15203, "Illegal Host for Token(Token非法主机)!"), - INVALID_TOKEN(15204, "Invalid Token(令牌无效)"); + INVALID_TOKEN(15204, "Invalid Token(令牌无效)"), + TOKEN_IS_NULL(15205, "token is null(token 令牌为空)!"), + FAILED_TO_BAD_SQLGRAMMAR(15206, "Failed to query token{0} data(Token:{0} 数据查询失败), Caused by:{1}"), + NOT_EXIST_DB(15207, "Token:{0} does not exist in the table(Token:{0} 表中不存在)!, Caused by:{1}"); /** (errorCode)错误码 */ private final int errorCode; diff --git a/linkis-spring-cloud-services/linkis-service-gateway/linkis-gateway-authentication/src/main/scala/org/apache/linkis/gateway/authentication/service/CachedTokenService.scala b/linkis-spring-cloud-services/linkis-service-gateway/linkis-gateway-authentication/src/main/scala/org/apache/linkis/gateway/authentication/service/CachedTokenService.scala index e839d0b5b5..b9b91d23e6 100644 --- a/linkis-spring-cloud-services/linkis-service-gateway/linkis-gateway-authentication/src/main/scala/org/apache/linkis/gateway/authentication/service/CachedTokenService.scala +++ b/linkis-spring-cloud-services/linkis-service-gateway/linkis-gateway-authentication/src/main/scala/org/apache/linkis/gateway/authentication/service/CachedTokenService.scala @@ -28,14 +28,15 @@ import org.apache.linkis.gateway.authentication.exception.{ TokenAuthException, TokenNotExistException } -import org.apache.linkis.gateway.authentication.exception.TokenNotExistException import org.springframework.beans.factory.annotation.Autowired import org.springframework.stereotype.Service +import java.text.MessageFormat import java.util.concurrent.{ExecutionException, TimeUnit} import com.google.common.cache.{CacheBuilder, CacheLoader, LoadingCache} +import com.google.common.util.concurrent.UncheckedExecutionException @Service class CachedTokenService extends TokenService { @@ -59,9 +60,9 @@ class CachedTokenService extends TokenService { }); -// def setTokenDao(tokenDao: TokenDao): Unit = { -// this.tokenDao = tokenDao -// } + // def setTokenDao(tokenDao: TokenDao): Unit = { + // this.tokenDao = tokenDao + // } /* TODO begin @@ -110,17 +111,26 @@ class CachedTokenService extends TokenService { t match { case x: ExecutionException => x.getCause match { - case _: TokenNotExistException => null - case _ => + case e: TokenNotExistException => + throw new TokenAuthException( + NOT_EXIST_DB.getErrorCode, + MessageFormat.format(NOT_EXIST_DB.getErrorDesc, tokenName, e.getMessage) + ) + case e => throw new TokenAuthException( FAILED_TO_LOAD_TOKEN.getErrorCode, - FAILED_TO_LOAD_TOKEN.getErrorDesc + MessageFormat.format(FAILED_TO_LOAD_TOKEN.getErrorDesc, tokenName, e.getMessage) ) } - case _ => + case e: UncheckedExecutionException => + throw new TokenAuthException( + FAILED_TO_BAD_SQLGRAMMAR.getErrorCode, + MessageFormat.format(FAILED_TO_BAD_SQLGRAMMAR.getErrorDesc, tokenName, e.getMessage) + ) + case e => throw new TokenAuthException( FAILED_TO_LOAD_TOKEN.getErrorCode, - FAILED_TO_LOAD_TOKEN.getErrorDesc + MessageFormat.format(FAILED_TO_LOAD_TOKEN.getErrorDesc, tokenName, e.getMessage) ) } ) From 462eadf0cf90880cfa74952f56775854972d498d Mon Sep 17 00:00:00 2001 From: ahaoyao Date: Tue, 11 Apr 2023 20:33:38 +0800 Subject: [PATCH 067/261] Basic Information Management Delete Interface Add Administrator Authentication --- .../server/restful/ErrorCodeRestfulApi.java | 8 ++++++-- .../server/restful/GatewayAuthTokenRestfulApi.java | 7 ++++++- .../restful/RmExternalResourceProviderRestfulApi.java | 9 +++++++-- 3 files changed, 19 insertions(+), 5 deletions(-) diff --git a/linkis-public-enhancements/linkis-basedata-manager/src/main/java/org/apache/linkis/basedatamanager/server/restful/ErrorCodeRestfulApi.java b/linkis-public-enhancements/linkis-basedata-manager/src/main/java/org/apache/linkis/basedatamanager/server/restful/ErrorCodeRestfulApi.java index 3c215a831b..1a24b36fd4 100644 --- a/linkis-public-enhancements/linkis-basedata-manager/src/main/java/org/apache/linkis/basedatamanager/server/restful/ErrorCodeRestfulApi.java +++ b/linkis-public-enhancements/linkis-basedata-manager/src/main/java/org/apache/linkis/basedatamanager/server/restful/ErrorCodeRestfulApi.java @@ -89,8 +89,12 @@ public Message add(HttpServletRequest request, @RequestBody ErrorCodeEntity erro @ApiOperation(value = "remove", notes = "Remove an Error Code by id", httpMethod = "DELETE") @RequestMapping(path = "/{id}", method = RequestMethod.DELETE) public Message remove(HttpServletRequest request, @PathVariable("id") Long id) { - ModuleUserUtils.getOperationUser( - request, "Remove a Datasource Code Record,id:" + id.toString()); + String username = + ModuleUserUtils.getOperationUser( + request, "Try to remove error code record with id:" + id.toString()); + if (!Configuration.isAdmin(username)) { + return Message.error("User '" + username + "' is not admin user[非管理员用户]"); + } boolean result = errorCodeService.removeById(id); return Message.ok("").data("result", result); } diff --git a/linkis-public-enhancements/linkis-basedata-manager/src/main/java/org/apache/linkis/basedatamanager/server/restful/GatewayAuthTokenRestfulApi.java b/linkis-public-enhancements/linkis-basedata-manager/src/main/java/org/apache/linkis/basedatamanager/server/restful/GatewayAuthTokenRestfulApi.java index 2b86e5bb43..7d5668c074 100644 --- a/linkis-public-enhancements/linkis-basedata-manager/src/main/java/org/apache/linkis/basedatamanager/server/restful/GatewayAuthTokenRestfulApi.java +++ b/linkis-public-enhancements/linkis-basedata-manager/src/main/java/org/apache/linkis/basedatamanager/server/restful/GatewayAuthTokenRestfulApi.java @@ -127,7 +127,12 @@ public Message update(HttpServletRequest request, @RequestBody GatewayAuthTokenE httpMethod = "DELETE") @RequestMapping(path = "/{id}", method = RequestMethod.DELETE) public Message remove(HttpServletRequest request, @PathVariable("id") Long id) { - ModuleUserUtils.getOperationUser(request, "Remove a Gateway Auth Token Record,id:" + id); + String username = + ModuleUserUtils.getOperationUser( + request, "Try to remove gateway auto token record with id:" + id); + if (!Configuration.isAdmin(username)) { + return Message.error("User '" + username + "' is not admin user[非管理员用户]"); + } boolean result = gatewayAuthTokenService.removeById(id); return Message.ok("").data("result", result); } diff --git a/linkis-public-enhancements/linkis-basedata-manager/src/main/java/org/apache/linkis/basedatamanager/server/restful/RmExternalResourceProviderRestfulApi.java b/linkis-public-enhancements/linkis-basedata-manager/src/main/java/org/apache/linkis/basedatamanager/server/restful/RmExternalResourceProviderRestfulApi.java index 5575eca20b..8b06b2e63f 100644 --- a/linkis-public-enhancements/linkis-basedata-manager/src/main/java/org/apache/linkis/basedatamanager/server/restful/RmExternalResourceProviderRestfulApi.java +++ b/linkis-public-enhancements/linkis-basedata-manager/src/main/java/org/apache/linkis/basedatamanager/server/restful/RmExternalResourceProviderRestfulApi.java @@ -112,8 +112,13 @@ public Message add( httpMethod = "DELETE") @RequestMapping(path = "/{id}", method = RequestMethod.DELETE) public Message remove(HttpServletRequest request, @PathVariable("id") Long id) { - ModuleUserUtils.getOperationUser( - request, "Remove a Resource manager External Resource Provider Record,id:" + id.toString()); + String username = + ModuleUserUtils.getOperationUser( + request, + "Try to remove resource external resource provider record with id:" + id.toString()); + if (!Configuration.isAdmin(username)) { + return Message.error("User '" + username + "' is not admin user[非管理员用户]"); + } boolean result = rmExternalResourceProviderService.removeById(id); return Message.ok("").data("result", result); } From 8634d6bf826abbf7505d01c559badfb697791aca Mon Sep 17 00:00:00 2001 From: huangKai-2323 <62878639+huangKai-2323@users.noreply.github.com> Date: Mon, 24 Apr 2023 14:49:34 +0800 Subject: [PATCH 068/261] 1. Add the eclist interface input parameter: queueName 2. Add the eclist interface parameter: lastUnlockTimestamp --- .../common/constant/ec/ECConstants.scala | 1 + .../ComputationEngineConnMetrics.scala | 8 +++ .../DefaultNodeHeartbeatMsgManager.scala | 5 ++ .../am/restful/ECResourceInfoRestfulApi.java | 21 +++++-- .../am/service/ECResourceInfoService.java | 6 +- .../impl/ECResourceInfoServiceImpl.java | 61 +++++++++++++------ .../persistence/PersistencerEcNodeInfo.java | 15 ++++- .../mapper/common/NodeManagerMapper.xml | 2 +- 8 files changed, 92 insertions(+), 27 deletions(-) diff --git a/linkis-computation-governance/linkis-computation-governance-common/src/main/scala/org/apache/linkis/governance/common/constant/ec/ECConstants.scala b/linkis-computation-governance/linkis-computation-governance-common/src/main/scala/org/apache/linkis/governance/common/constant/ec/ECConstants.scala index fe48f6887d..b065ad03e9 100644 --- a/linkis-computation-governance/linkis-computation-governance-common/src/main/scala/org/apache/linkis/governance/common/constant/ec/ECConstants.scala +++ b/linkis-computation-governance/linkis-computation-governance-common/src/main/scala/org/apache/linkis/governance/common/constant/ec/ECConstants.scala @@ -35,4 +35,5 @@ object ECConstants { val YARN_QUEUE_NAME_CONFIG_KEY = "wds.linkis.rm.yarnqueue" + val EC_LAST_UNLOCK_TIME_MILLS = "lastUnlockTimestamp" } diff --git a/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/metrics/ComputationEngineConnMetrics.scala b/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/metrics/ComputationEngineConnMetrics.scala index f96896f557..d5ac6e3f58 100644 --- a/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/metrics/ComputationEngineConnMetrics.scala +++ b/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/metrics/ComputationEngineConnMetrics.scala @@ -91,4 +91,12 @@ object ComputationEngineConnMetrics { getTotalBusyTimeMills(nodeStatus) + getTotalIdleTimeMills(nodeStatus) def getUnlockToShutdownDurationMills(): Long = unlockToShutdownDurationMills.get() + + def getLastUnlockTimeMills(nodeStatus: NodeStatus): Long = { + nodeStatus match { + case NodeStatus.Unlock => lastUnlockTimeMills + case _ => 0 + } + } + } diff --git a/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/service/DefaultNodeHeartbeatMsgManager.scala b/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/service/DefaultNodeHeartbeatMsgManager.scala index eccf54bfad..c6b0020030 100644 --- a/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/service/DefaultNodeHeartbeatMsgManager.scala +++ b/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/service/DefaultNodeHeartbeatMsgManager.scala @@ -23,6 +23,7 @@ import org.apache.linkis.engineconn.computation.executor.metrics.ComputationEngi import org.apache.linkis.engineconn.core.EngineConnObject import org.apache.linkis.engineconn.executor.entity.{Executor, SensibleExecutor} import org.apache.linkis.governance.common.constant.ec.ECConstants +import org.apache.linkis.manager.common.entity.enumeration.NodeStatus import org.apache.linkis.server.BDPJettyServerHelper import org.springframework.stereotype.Component @@ -63,6 +64,10 @@ class DefaultNodeHeartbeatMsgManager extends NodeHeartbeatMsgManager with Loggin ECConstants.EC_TOTAL_LOCK_TIME_MILLS_KEY, ComputationEngineConnMetrics.getTotalLockTimeMills(status).asInstanceOf[Object] ) + msgMap.put( + ECConstants.EC_LAST_UNLOCK_TIME_MILLS, + ComputationEngineConnMetrics.getLastUnlockTimeMills(status).asInstanceOf[Object] + ) case _ => } val engineParams = EngineConnObject.getEngineCreationContext.getOptions diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/restful/ECResourceInfoRestfulApi.java b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/restful/ECResourceInfoRestfulApi.java index 03a2b1465d..26e53cc351 100644 --- a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/restful/ECResourceInfoRestfulApi.java +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/restful/ECResourceInfoRestfulApi.java @@ -188,6 +188,7 @@ public Message queryEcrHistory( @ApiImplicitParam(name = "creators", dataType = "Array", required = true, value = "creators"), @ApiImplicitParam(name = "engineTypes", dataType = "Array", value = "engine type"), @ApiImplicitParam(name = "statuss", dataType = "Array", value = "statuss"), + @ApiImplicitParam(name = "queueName", dataType = "String", value = "queueName"), }) @RequestMapping(path = "/ecList", method = RequestMethod.POST) public Message queryEcList(HttpServletRequest req, @RequestBody JsonNode jsonNode) { @@ -195,6 +196,7 @@ public Message queryEcList(HttpServletRequest req, @RequestBody JsonNode jsonNod JsonNode creatorsParam = jsonNode.get("creators"); JsonNode engineTypesParam = jsonNode.get("engineTypes"); JsonNode statussParam = jsonNode.get("statuss"); + JsonNode queueNameParam = jsonNode.get("queueName"); if (creatorsParam == null || creatorsParam.isNull() || creatorsParam.size() == 0) { return Message.error("creators is null in the parameters of the request(请求参数中【creators】为空)"); @@ -230,7 +232,16 @@ public Message queryEcList(HttpServletRequest req, @RequestBody JsonNode jsonNod return Message.error("parameters:statuss parsing failed(请求参数【statuss】解析失败)"); } } - + String queueName = ""; + if (queueNameParam != null && !queueNameParam.isNull()) { + try { + queueName = + JsonUtils.jackson() + .readValue(queueNameParam.toString(), new TypeReference() {}); + } catch (JsonProcessingException e) { + return Message.error("parameters:queueName parsing failed(请求参数【queueName】解析失败)"); + } + } String username = ModuleUserUtils.getOperationUser(req, "ecList"); String token = ModuleUserUtils.getToken(req); @@ -252,13 +263,15 @@ public Message queryEcList(HttpServletRequest req, @RequestBody JsonNode jsonNod } logger.info( - "request parameters creatorUserList:[{}], engineTypeList:[{}], statusStrList:[{}]", + "request parameters creatorUserList:[{}], engineTypeList:[{}], statusStrList:[{}], queueName:{}", String.join(",", creatorUserList), String.join(",", engineTypeList), - String.join(",", statusStrList)); + String.join(",", statusStrList), + queueNameParam); List> list = - ecResourceInfoService.getECResourceInfoList(creatorUserList, engineTypeList, statusStrList); + ecResourceInfoService.getECResourceInfoList( + creatorUserList, engineTypeList, statusStrList, queueName); return Message.ok().data("ecList", list); } diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/service/ECResourceInfoService.java b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/service/ECResourceInfoService.java index 6aabe19198..03a2509fcb 100644 --- a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/service/ECResourceInfoService.java +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/service/ECResourceInfoService.java @@ -40,8 +40,12 @@ List getECResourceInfoRecordList( * @param creatorUserList engineconn creator list * @param engineTypeList engineconn type list * @param statusStrList engineconn status string list + * @param queueName * @return */ List> getECResourceInfoList( - List creatorUserList, List engineTypeList, List statusStrList); + List creatorUserList, + List engineTypeList, + List statusStrList, + String queueName); } diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/service/impl/ECResourceInfoServiceImpl.java b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/service/impl/ECResourceInfoServiceImpl.java index e8988fee9b..af1f04f2c6 100644 --- a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/service/impl/ECResourceInfoServiceImpl.java +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/service/impl/ECResourceInfoServiceImpl.java @@ -28,12 +28,15 @@ import org.apache.linkis.manager.dao.NodeManagerMapper; import org.apache.linkis.manager.label.service.NodeLabelService; import org.apache.linkis.manager.persistence.LabelManagerPersistence; +import org.apache.linkis.server.BDPJettyServerHelper; +import org.apache.commons.collections.MapUtils; import org.apache.commons.lang3.StringUtils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; +import java.math.BigDecimal; import java.text.SimpleDateFormat; import java.util.*; import java.util.stream.Collectors; @@ -101,7 +104,10 @@ public List getECResourceInfoRecordList( @Override public List> getECResourceInfoList( - List creatorUserList, List engineTypeList, List statusStrList) { + List creatorUserList, + List engineTypeList, + List statusStrList, + String queueName) { List> resultList = new ArrayList<>(); @@ -138,26 +144,41 @@ public List> getECResourceInfoList( logger.info("Can not get any ec node info of ec:{}", ecNodeinfo.getInstance()); } else { try { - Map item = - json.readValue( - json.writeValueAsString(ecNodeinfo), - new TypeReference>() {}); - - Integer intStatus = ecNodeinfo.getInstanceStatus(); - item.put("instanceStatus", NodeStatus.values()[intStatus].name()); - String usedResourceStr = latestRecord.getUsedResource(); - /* - {"instance":1,"memory":"2.0 GB","cpu":1} - -> - {"driver":{"instance":1,"memory":"2.0 GB","cpu":1} } - */ - - item.put("useResource", ECResourceInfoUtils.getStringToMap(usedResourceStr)); - item.put("ecmInstance", latestRecord.getEcmInstance()); - String engineType = latestRecord.getLabelValue().split(",")[1].split("-")[0]; - item.put("engineType", engineType); - resultList.add(item); + Map usedResourceMap = + ECResourceInfoUtils.getStringToMap(usedResourceStr); + Map yarn = MapUtils.getMap(usedResourceMap, "yarn"); + String queueNameStr = String.valueOf(yarn.get("queueName")); + if ((StringUtils.isNotBlank(queueName) + && StringUtils.isNotBlank(queueNameStr) + && queueName.equals(queueNameStr)) + || StringUtils.isBlank(queueName)) { + Map item = + json.readValue( + json.writeValueAsString(ecNodeinfo), + new TypeReference>() {}); + + Integer intStatus = ecNodeinfo.getInstanceStatus(); + item.put("instanceStatus", NodeStatus.values()[intStatus].name()); + /* + {"instance":1,"memory":"2.0 GB","cpu":1} + -> + {"driver":{"instance":1,"memory":"2.0 GB","cpu":1} } + */ + HashMap heartbeatMap = + BDPJettyServerHelper.gson() + .fromJson(ecNodeinfo.getHeartbeatMsg(), new HashMap<>().getClass()); + Object lastUnlockTimeMills1 = + Optional.ofNullable(heartbeatMap.get("lastUnlockTimestamp")).orElse(0); + BigDecimal lastUnlockTimeMills = + new BigDecimal(String.valueOf(lastUnlockTimeMills1)); + item.put("lastUnlockTimestamp", lastUnlockTimeMills.longValue()); + item.put("useResource", ECResourceInfoUtils.getStringToMap(usedResourceStr)); + item.put("ecmInstance", latestRecord.getEcmInstance()); + String engineType = latestRecord.getLabelValue().split(",")[1].split("-")[0]; + item.put("engineType", engineType); + resultList.add(item); + } } catch (JsonProcessingException e) { logger.error("Fail to process the ec node info: [{}]", ecNodeinfo, e); } diff --git a/linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/java/org/apache/linkis/manager/common/entity/persistence/PersistencerEcNodeInfo.java b/linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/java/org/apache/linkis/manager/common/entity/persistence/PersistencerEcNodeInfo.java index 11665fb851..2578433052 100644 --- a/linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/java/org/apache/linkis/manager/common/entity/persistence/PersistencerEcNodeInfo.java +++ b/linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/java/org/apache/linkis/manager/common/entity/persistence/PersistencerEcNodeInfo.java @@ -24,6 +24,8 @@ public class PersistencerEcNodeInfo extends PersistenceNode { private String engineType; + private String heartbeatMsg; + public Integer getInstanceStatus() { return instanceStatus; } @@ -40,6 +42,14 @@ public void setEngineType(String engineType) { this.engineType = engineType; } + public String getHeartbeatMsg() { + return heartbeatMsg; + } + + public void setHeartbeatMsg(String heartbeatMsg) { + this.heartbeatMsg = heartbeatMsg; + } + @Override public String toString() { return "PersistencerEcNodeInfo{" @@ -48,7 +58,10 @@ public String toString() { + ", engineType='" + engineType + '\'' - + "} " + + ", heartbeatMsg='" + + heartbeatMsg + + '\'' + + '}' + super.toString(); } } diff --git a/linkis-computation-governance/linkis-manager/linkis-manager-persistence/src/main/resources/mapper/common/NodeManagerMapper.xml b/linkis-computation-governance/linkis-manager/linkis-manager-persistence/src/main/resources/mapper/common/NodeManagerMapper.xml index 2294cdbca4..c2626d0f64 100644 --- a/linkis-computation-governance/linkis-manager/linkis-manager-persistence/src/main/resources/mapper/common/NodeManagerMapper.xml +++ b/linkis-computation-governance/linkis-manager/linkis-manager-persistence/src/main/resources/mapper/common/NodeManagerMapper.xml @@ -118,7 +118,7 @@ From f9b06febf4ca836e359f97e55c8a8c6f41eaeca2 Mon Sep 17 00:00:00 2001 From: huangKai-2323 <62878639+huangKai-2323@users.noreply.github.com> Date: Tue, 25 Apr 2023 18:08:51 +0800 Subject: [PATCH 075/261] Code Formatting --- .../am/restful/ECResourceInfoRestfulApi.java | 8 ++++---- .../am/service/ECResourceInfoService.java | 9 +++++---- .../service/impl/ECResourceInfoServiceImpl.java | 16 ++++++++-------- .../linkis/manager/dao/NodeManagerMapper.java | 4 +++- 4 files changed, 20 insertions(+), 17 deletions(-) diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/restful/ECResourceInfoRestfulApi.java b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/restful/ECResourceInfoRestfulApi.java index 5ff74c4142..24292a6817 100644 --- a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/restful/ECResourceInfoRestfulApi.java +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/restful/ECResourceInfoRestfulApi.java @@ -267,14 +267,14 @@ public Message queryEcList(HttpServletRequest req, @RequestBody JsonNode jsonNod if (instanceNameParam != null && !instanceNameParam.isNull()) { try { instanceNameList = - JsonUtils.jackson() - .readValue(instanceNameParam.toString(), new TypeReference>() {}); + JsonUtils.jackson() + .readValue(instanceNameParam.toString(), new TypeReference>() {}); } catch (JsonProcessingException e) { return Message.error("parameters:instanceName parsing failed(请求参数【instanceName】解析失败)"); } } logger.info( - "request parameters creatorUserList:[{}], engineTypeList:[{}], statusStrList:[{}], queueName:{}", + "request parameters creatorUserList:[{}], engineTypeList:[{}], statusStrList:[{}], queueName:{}, instanceNameList:{}", String.join(",", creatorUserList), String.join(",", engineTypeList), String.join(",", statusStrList), @@ -283,7 +283,7 @@ public Message queryEcList(HttpServletRequest req, @RequestBody JsonNode jsonNod List> list = ecResourceInfoService.getECResourceInfoList( - creatorUserList, engineTypeList, statusStrList, queueName,instanceNameList); + creatorUserList, engineTypeList, statusStrList, queueName, instanceNameList); return Message.ok().data("ecList", list); } diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/service/ECResourceInfoService.java b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/service/ECResourceInfoService.java index 0738b38a8e..b7ac14866e 100644 --- a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/service/ECResourceInfoService.java +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/service/ECResourceInfoService.java @@ -45,8 +45,9 @@ List getECResourceInfoRecordList( * @return */ List> getECResourceInfoList( - List creatorUserList, - List engineTypeList, - List statusStrList, - String queueName, List instanceNameList); + List creatorUserList, + List engineTypeList, + List statusStrList, + String queueName, + List instanceNameList); } diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/service/impl/ECResourceInfoServiceImpl.java b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/service/impl/ECResourceInfoServiceImpl.java index c66f0b4da0..d910d32ebc 100644 --- a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/service/impl/ECResourceInfoServiceImpl.java +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/service/impl/ECResourceInfoServiceImpl.java @@ -104,11 +104,11 @@ public List getECResourceInfoRecordList( @Override public List> getECResourceInfoList( - List creatorUserList, - List engineTypeList, - List statusStrList, - String queueName, - List instanceNameList) { + List creatorUserList, + List engineTypeList, + List statusStrList, + String queueName, + List instanceNameList) { List> resultList = new ArrayList<>(); @@ -120,7 +120,7 @@ public List> getECResourceInfoList( // get engine conn info list filter by creator user list /instance status list List ecNodesInfo = - nodeManagerMapper.getEMNodeInfoList(creatorUserList, statusIntList,instanceNameList); + nodeManagerMapper.getEMNodeInfoList(creatorUserList, statusIntList, instanceNameList); // map k:v---> instanceName：PersistencerEcNodeInfo Map persistencerEcNodeInfoMap = @@ -148,8 +148,8 @@ public List> getECResourceInfoList( String usedResourceStr = latestRecord.getUsedResource(); Map usedResourceMap = ECResourceInfoUtils.getStringToMap(usedResourceStr); - Map yarn = MapUtils.getMap(usedResourceMap, "yarn",new HashMap()); - String queueNameStr = String.valueOf(yarn.getOrDefault("queueName","")); + Map yarn = MapUtils.getMap(usedResourceMap, "yarn", new HashMap()); + String queueNameStr = String.valueOf(yarn.getOrDefault("queueName", "")); if ((StringUtils.isNotBlank(queueName) && StringUtils.isNotBlank(queueNameStr) && queueName.equals(queueNameStr)) diff --git a/linkis-computation-governance/linkis-manager/linkis-manager-persistence/src/main/java/org/apache/linkis/manager/dao/NodeManagerMapper.java b/linkis-computation-governance/linkis-manager/linkis-manager-persistence/src/main/java/org/apache/linkis/manager/dao/NodeManagerMapper.java index 2f7b251da0..910102895c 100644 --- a/linkis-computation-governance/linkis-manager/linkis-manager-persistence/src/main/java/org/apache/linkis/manager/dao/NodeManagerMapper.java +++ b/linkis-computation-governance/linkis-manager/linkis-manager-persistence/src/main/java/org/apache/linkis/manager/dao/NodeManagerMapper.java @@ -77,5 +77,7 @@ void updateNodeLabelRelation( List getNodeInstancesByOwnerList(@Param("owner") List owner); List getEMNodeInfoList( - @Param("creatorUsers") List creatorUsers, @Param("statuss") List statuss,@Param("instanceName") List instanceNameList); + @Param("creatorUsers") List creatorUsers, + @Param("statuss") List statuss, + @Param("instanceName") List instanceNameList); } From a2f118d5c84bb88d8e59eede3e61be04746336a2 Mon Sep 17 00:00:00 2001 From: casionone Date: Tue, 25 Apr 2023 18:58:01 +0800 Subject: [PATCH 076/261] adjust dss gateway support --- .../linkis-gateway-server-support/pom.xml | 16 +------------- .../src/main/assembly/distribution.xml | 5 ----- .../DSSGatewayConfiguration.scala | 2 +- .../dss/{ => parser}/DSSGatewayParser.scala | 21 +++++++++---------- .../{ => parser}/DSSRouteLabelParser.scala | 8 +------ 5 files changed, 13 insertions(+), 39 deletions(-) rename linkis-spring-cloud-services/linkis-service-gateway/linkis-gateway-server-support/src/main/scala/org/apache/linkis/gateway/dss/{ => parser}/DSSGatewayConfiguration.scala (97%) rename linkis-spring-cloud-services/linkis-service-gateway/linkis-gateway-server-support/src/main/scala/org/apache/linkis/gateway/dss/{ => parser}/DSSGatewayParser.scala (95%) rename linkis-spring-cloud-services/linkis-service-gateway/linkis-gateway-server-support/src/main/scala/org/apache/linkis/gateway/dss/{ => parser}/DSSRouteLabelParser.scala (90%) diff --git a/linkis-spring-cloud-services/linkis-service-gateway/linkis-gateway-server-support/pom.xml b/linkis-spring-cloud-services/linkis-service-gateway/linkis-gateway-server-support/pom.xml index ef4635ae00..8d8c131abb 100644 --- a/linkis-spring-cloud-services/linkis-service-gateway/linkis-gateway-server-support/pom.xml +++ b/linkis-spring-cloud-services/linkis-service-gateway/linkis-gateway-server-support/pom.xml @@ -94,19 +94,6 @@ jackson-databind - - - com.webank.wedatasphere.dss - dss-gateway-support - 1.1.1 - - - org.apache.linkis - linkis-gateway-server-support - - - - @@ -115,7 +102,6 @@ net.alchim31.maven scala-maven-plugin - org.apache.maven.plugins maven-assembly-plugin @@ -126,7 +112,7 @@ false false - /src/main/assembly/distribution.xml + src/main/assembly/distribution.xml diff --git a/linkis-spring-cloud-services/linkis-service-gateway/linkis-gateway-server-support/src/main/assembly/distribution.xml b/linkis-spring-cloud-services/linkis-service-gateway/linkis-gateway-server-support/src/main/assembly/distribution.xml index 7180fea9ce..cf2fd12322 100644 --- a/linkis-spring-cloud-services/linkis-service-gateway/linkis-gateway-server-support/src/main/assembly/distribution.xml +++ b/linkis-spring-cloud-services/linkis-service-gateway/linkis-gateway-server-support/src/main/assembly/distribution.xml @@ -21,7 +21,6 @@ linkis-gateway dir - zip false linkis-gateway @@ -44,9 +43,5 @@ - - - - diff --git a/linkis-spring-cloud-services/linkis-service-gateway/linkis-gateway-server-support/src/main/scala/org/apache/linkis/gateway/dss/DSSGatewayConfiguration.scala b/linkis-spring-cloud-services/linkis-service-gateway/linkis-gateway-server-support/src/main/scala/org/apache/linkis/gateway/dss/parser/DSSGatewayConfiguration.scala similarity index 97% rename from linkis-spring-cloud-services/linkis-service-gateway/linkis-gateway-server-support/src/main/scala/org/apache/linkis/gateway/dss/DSSGatewayConfiguration.scala rename to linkis-spring-cloud-services/linkis-service-gateway/linkis-gateway-server-support/src/main/scala/org/apache/linkis/gateway/dss/parser/DSSGatewayConfiguration.scala index a649e9c182..937b0ddeb7 100644 --- a/linkis-spring-cloud-services/linkis-service-gateway/linkis-gateway-server-support/src/main/scala/org/apache/linkis/gateway/dss/DSSGatewayConfiguration.scala +++ b/linkis-spring-cloud-services/linkis-service-gateway/linkis-gateway-server-support/src/main/scala/org/apache/linkis/gateway/dss/parser/DSSGatewayConfiguration.scala @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.linkis.gateway.dss +package org.apache.linkis.gateway.dss.parser import org.apache.linkis.common.conf.CommonVars diff --git a/linkis-spring-cloud-services/linkis-service-gateway/linkis-gateway-server-support/src/main/scala/org/apache/linkis/gateway/dss/DSSGatewayParser.scala b/linkis-spring-cloud-services/linkis-service-gateway/linkis-gateway-server-support/src/main/scala/org/apache/linkis/gateway/dss/parser/DSSGatewayParser.scala similarity index 95% rename from linkis-spring-cloud-services/linkis-service-gateway/linkis-gateway-server-support/src/main/scala/org/apache/linkis/gateway/dss/DSSGatewayParser.scala rename to linkis-spring-cloud-services/linkis-service-gateway/linkis-gateway-server-support/src/main/scala/org/apache/linkis/gateway/dss/parser/DSSGatewayParser.scala index cca49edcdc..0423304129 100644 --- a/linkis-spring-cloud-services/linkis-service-gateway/linkis-gateway-server-support/src/main/scala/org/apache/linkis/gateway/dss/DSSGatewayParser.scala +++ b/linkis-spring-cloud-services/linkis-service-gateway/linkis-gateway-server-support/src/main/scala/org/apache/linkis/gateway/dss/parser/DSSGatewayParser.scala @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.linkis.gateway.dss +package org.apache.linkis.gateway.dss.parser import org.apache.linkis.common.ServiceInstance import org.apache.linkis.gateway.exception.TooManyServiceException @@ -38,7 +38,7 @@ import org.springframework.stereotype.Component import java.util import java.util.Locale -import scala.collection.JavaConversions._ +import scala.collection.JavaConverters._ @Component class DSSGatewayParser extends AbstractGatewayParser { @@ -162,7 +162,7 @@ class DSSGatewayParser extends AbstractGatewayParser { logger.info( "Get ServiceName From Label and method is " + gatewayContext.getRequest.getMethod.toString + ",and urlLabels is " + requestUrlLabels ) - val requestMethod = gatewayContext.getRequest.getMethod.toLowerCase(Locale.ROOT) + val requestMethod = gatewayContext.getRequest.getMethod.toLowerCase(Locale.getDefault()) if ( requestUrlLabels == null && (requestMethod .equals("post") || requestMethod.equals("put") || requestMethod.equals("delete")) @@ -181,7 +181,7 @@ class DSSGatewayParser extends AbstractGatewayParser { case map: util.Map[String, Any] => labelBuilderFactory.getLabels(map.asInstanceOf) case _ => new util.ArrayList[Label[_]]() } - labels + labels.asScala .filter(label => label.isInstanceOf[RouteLabel]) .foreach(label => { routeLabelList.add(label.asInstanceOf[RouteLabel]) @@ -189,7 +189,7 @@ class DSSGatewayParser extends AbstractGatewayParser { case _ => null } - val labelNameList = routeLabelList.map(routeLabel => routeLabel.getStringValue).toList + val labelNameList = routeLabelList.asScala.map(routeLabel => routeLabel.getStringValue).toList if (labelNameList != null && labelNameList.size > 0) { genServiceNameByDSSLabel(labelNameList, tmpServiceName) } else if (null != requestUrlLabels) { @@ -241,7 +241,7 @@ class DSSGatewayParser extends AbstractGatewayParser { ): Option[String] = { val findIt: (String => Boolean) => Option[String] = op => { val services = - SpringCloudFeignConfigurationCache.getDiscoveryClient.getServices.filter(op).toList + SpringCloudFeignConfigurationCache.getDiscoveryClient.getServices.asScala.filter(op).toList if (services.length == 1) Some(services.head) else if (services.length > 1) tooManyDeal(services) else None @@ -252,7 +252,7 @@ class DSSGatewayParser extends AbstractGatewayParser { val findMostCorrect: (String => (String, Int)) => Option[String] = { op => { val serviceMap = - SpringCloudFeignConfigurationCache.getDiscoveryClient.getServices.map(op).toMap + SpringCloudFeignConfigurationCache.getDiscoveryClient.getServices.asScala.map(op).toMap var count = 0 var retService: Option[String] = None serviceMap.foreach { case (k, v) => @@ -266,10 +266,9 @@ class DSSGatewayParser extends AbstractGatewayParser { retService } } - val lowerServiceId = parsedServiceId.toLowerCase() - val serverName = tmpServerName.toLowerCase() - // findIt(_.toLowerCase() == lowerServiceId).orElse(findIt(_.toLowerCase().contains(lowerServiceId))) - findIt(_.toLowerCase() == serverName).orElse(findMostCorrect(service => { + val lowerServiceId = parsedServiceId.toLowerCase(Locale.getDefault()) + val serverName = tmpServerName.toLowerCase(Locale.getDefault()) + findIt(_.toLowerCase(Locale.getDefault()) == serverName).orElse(findMostCorrect(service => { (service, lowerServiceId.split("/").count(word => service.contains(word))) })) } diff --git a/linkis-spring-cloud-services/linkis-service-gateway/linkis-gateway-server-support/src/main/scala/org/apache/linkis/gateway/dss/DSSRouteLabelParser.scala b/linkis-spring-cloud-services/linkis-service-gateway/linkis-gateway-server-support/src/main/scala/org/apache/linkis/gateway/dss/parser/DSSRouteLabelParser.scala similarity index 90% rename from linkis-spring-cloud-services/linkis-service-gateway/linkis-gateway-server-support/src/main/scala/org/apache/linkis/gateway/dss/DSSRouteLabelParser.scala rename to linkis-spring-cloud-services/linkis-service-gateway/linkis-gateway-server-support/src/main/scala/org/apache/linkis/gateway/dss/parser/DSSRouteLabelParser.scala index 2587c823c6..ddf87c3258 100644 --- a/linkis-spring-cloud-services/linkis-service-gateway/linkis-gateway-server-support/src/main/scala/org/apache/linkis/gateway/dss/DSSRouteLabelParser.scala +++ b/linkis-spring-cloud-services/linkis-service-gateway/linkis-gateway-server-support/src/main/scala/org/apache/linkis/gateway/dss/parser/DSSRouteLabelParser.scala @@ -15,22 +15,16 @@ * limitations under the License. */ -package org.apache.linkis.gateway.dss +package org.apache.linkis.gateway.dss.parser import org.apache.linkis.gateway.http.GatewayContext import org.apache.linkis.gateway.ujes.route.label.RouteLabelParser -import org.apache.linkis.manager.label.builder.factory.LabelBuilderFactoryContext -import org.apache.linkis.manager.label.entity.Label import org.apache.linkis.manager.label.entity.route.RouteLabel -import org.apache.linkis.protocol.constants.TaskConstant -import org.apache.linkis.server.BDPJettyServerHelper import org.springframework.stereotype.Component import java.util -import scala.collection.JavaConversions._ - @Component class DSSRouteLabelParser extends RouteLabelParser { From 035dc65de1abb7b3a52819c4e3a454d4dd7566c7 Mon Sep 17 00:00:00 2001 From: huangKai-2323 <62878639+huangKai-2323@users.noreply.github.com> Date: Wed, 26 Apr 2023 11:17:59 +0800 Subject: [PATCH 077/261] Code Formatting --- .../common/constant/ec/ECConstants.scala | 2 +- .../ComputationEngineConnMetrics.scala | 2 +- .../DefaultNodeHeartbeatMsgManager.scala | 2 +- .../am/restful/ECResourceInfoRestfulApi.java | 15 ++--- .../impl/ECResourceInfoServiceImpl.java | 56 ++++++++++--------- 5 files changed, 42 insertions(+), 35 deletions(-) diff --git a/linkis-computation-governance/linkis-computation-governance-common/src/main/scala/org/apache/linkis/governance/common/constant/ec/ECConstants.scala b/linkis-computation-governance/linkis-computation-governance-common/src/main/scala/org/apache/linkis/governance/common/constant/ec/ECConstants.scala index b065ad03e9..c3dfbed357 100644 --- a/linkis-computation-governance/linkis-computation-governance-common/src/main/scala/org/apache/linkis/governance/common/constant/ec/ECConstants.scala +++ b/linkis-computation-governance/linkis-computation-governance-common/src/main/scala/org/apache/linkis/governance/common/constant/ec/ECConstants.scala @@ -35,5 +35,5 @@ object ECConstants { val YARN_QUEUE_NAME_CONFIG_KEY = "wds.linkis.rm.yarnqueue" - val EC_LAST_UNLOCK_TIME_MILLS = "lastUnlockTimestamp" + val EC_LAST_UNLOCK_TIMESTAMP = "lastUnlockTimestamp" } diff --git a/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/metrics/ComputationEngineConnMetrics.scala b/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/metrics/ComputationEngineConnMetrics.scala index d5ac6e3f58..4446bdc672 100644 --- a/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/metrics/ComputationEngineConnMetrics.scala +++ b/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/metrics/ComputationEngineConnMetrics.scala @@ -92,7 +92,7 @@ object ComputationEngineConnMetrics { def getUnlockToShutdownDurationMills(): Long = unlockToShutdownDurationMills.get() - def getLastUnlockTimeMills(nodeStatus: NodeStatus): Long = { + def getLastUnlockTimestamp(nodeStatus: NodeStatus): Long = { nodeStatus match { case NodeStatus.Unlock => lastUnlockTimeMills case _ => 0 diff --git a/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/service/DefaultNodeHeartbeatMsgManager.scala b/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/service/DefaultNodeHeartbeatMsgManager.scala index c6b0020030..b27aad9d95 100644 --- a/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/service/DefaultNodeHeartbeatMsgManager.scala +++ b/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/service/DefaultNodeHeartbeatMsgManager.scala @@ -66,7 +66,7 @@ class DefaultNodeHeartbeatMsgManager extends NodeHeartbeatMsgManager with Loggin ) msgMap.put( ECConstants.EC_LAST_UNLOCK_TIME_MILLS, - ComputationEngineConnMetrics.getLastUnlockTimeMills(status).asInstanceOf[Object] + ComputationEngineConnMetrics.getLastUnlockTimestamp(status).asInstanceOf[Object] ) case _ => } diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/restful/ECResourceInfoRestfulApi.java b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/restful/ECResourceInfoRestfulApi.java index 24292a6817..e00a3c7bd8 100644 --- a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/restful/ECResourceInfoRestfulApi.java +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/restful/ECResourceInfoRestfulApi.java @@ -189,6 +189,7 @@ public Message queryEcrHistory( @ApiImplicitParam(name = "engineTypes", dataType = "Array", value = "engine type"), @ApiImplicitParam(name = "statuss", dataType = "Array", value = "statuss"), @ApiImplicitParam(name = "queueName", dataType = "String", value = "queueName"), + @ApiImplicitParam(name = "ecInstances", dataType = "String", value = "ecInstances"), }) @RequestMapping(path = "/ecList", method = RequestMethod.POST) public Message queryEcList(HttpServletRequest req, @RequestBody JsonNode jsonNode) { @@ -209,7 +210,7 @@ public Message queryEcList(HttpServletRequest req, @RequestBody JsonNode jsonNod JsonNode engineTypesParam = jsonNode.get("engineTypes"); JsonNode statussParam = jsonNode.get("statuss"); JsonNode queueNameParam = jsonNode.get("queueName"); - JsonNode instanceNameParam = jsonNode.get("instanceName"); + JsonNode ecInstancesParam = jsonNode.get("ecInstances"); // if (creatorsParam == null || creatorsParam.isNull() || creatorsParam.size() == 0) { // return Message.error("creators is null in the parameters of the @@ -263,12 +264,12 @@ public Message queryEcList(HttpServletRequest req, @RequestBody JsonNode jsonNod return Message.error("parameters:queueName parsing failed(请求参数【queueName】解析失败)"); } } - List instanceNameList = new ArrayList<>(); - if (instanceNameParam != null && !instanceNameParam.isNull()) { + List ecInstancesList = new ArrayList<>(); + if (ecInstancesParam != null && !ecInstancesParam.isNull()) { try { - instanceNameList = + ecInstancesList = JsonUtils.jackson() - .readValue(instanceNameParam.toString(), new TypeReference>() {}); + .readValue(ecInstancesParam.toString(), new TypeReference>() {}); } catch (JsonProcessingException e) { return Message.error("parameters:instanceName parsing failed(请求参数【instanceName】解析失败)"); } @@ -278,12 +279,12 @@ public Message queryEcList(HttpServletRequest req, @RequestBody JsonNode jsonNod String.join(",", creatorUserList), String.join(",", engineTypeList), String.join(",", statusStrList), - String.join(",", instanceNameList), + String.join(",", ecInstancesList), queueNameParam); List> list = ecResourceInfoService.getECResourceInfoList( - creatorUserList, engineTypeList, statusStrList, queueName, instanceNameList); + creatorUserList, engineTypeList, statusStrList, queueName, ecInstancesList); return Message.ok().data("ecList", list); } diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/service/impl/ECResourceInfoServiceImpl.java b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/service/impl/ECResourceInfoServiceImpl.java index d910d32ebc..3e8bc0c441 100644 --- a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/service/impl/ECResourceInfoServiceImpl.java +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/service/impl/ECResourceInfoServiceImpl.java @@ -145,27 +145,22 @@ public List> getECResourceInfoList( logger.info("Can not get any ec node info of ec:{}", ecNodeinfo.getInstance()); } else { try { + Map item = + json.readValue( + json.writeValueAsString(ecNodeinfo), + new TypeReference>() {}); + + Integer intStatus = ecNodeinfo.getInstanceStatus(); + item.put("instanceStatus", NodeStatus.values()[intStatus].name()); + String usedResourceStr = latestRecord.getUsedResource(); - Map usedResourceMap = - ECResourceInfoUtils.getStringToMap(usedResourceStr); - Map yarn = MapUtils.getMap(usedResourceMap, "yarn", new HashMap()); - String queueNameStr = String.valueOf(yarn.getOrDefault("queueName", "")); - if ((StringUtils.isNotBlank(queueName) - && StringUtils.isNotBlank(queueNameStr) - && queueName.equals(queueNameStr)) - || StringUtils.isBlank(queueName)) { - Map item = - json.readValue( - json.writeValueAsString(ecNodeinfo), - new TypeReference>() {}); - - Integer intStatus = ecNodeinfo.getInstanceStatus(); - item.put("instanceStatus", NodeStatus.values()[intStatus].name()); - /* - {"instance":1,"memory":"2.0 GB","cpu":1} - -> - {"driver":{"instance":1,"memory":"2.0 GB","cpu":1} } - */ + /* + {"instance":1,"memory":"2.0 GB","cpu":1} + -> + {"driver":{"instance":1,"memory":"2.0 GB","cpu":1} } + */ + long lastUnlockTimestamp = 0L; + if (NodeStatus.values()[intStatus].name().equals(NodeStatus.Unlock.name())) { HashMap heartbeatMap = BDPJettyServerHelper.gson() .fromJson(ecNodeinfo.getHeartbeatMsg(), new HashMap<>().getClass()); @@ -173,11 +168,22 @@ public List> getECResourceInfoList( Optional.ofNullable(heartbeatMap.get("lastUnlockTimestamp")).orElse(0); BigDecimal lastUnlockTimeMills = new BigDecimal(String.valueOf(lastUnlockTimeMills1)); - item.put("lastUnlockTimestamp", lastUnlockTimeMills.longValue()); - item.put("useResource", ECResourceInfoUtils.getStringToMap(usedResourceStr)); - item.put("ecmInstance", latestRecord.getEcmInstance()); - String engineType = latestRecord.getLabelValue().split(",")[1].split("-")[0]; - item.put("engineType", engineType); + lastUnlockTimestamp = lastUnlockTimeMills.longValue(); + } + item.put("lastUnlockTimestamp", lastUnlockTimestamp); + item.put("useResource", ECResourceInfoUtils.getStringToMap(usedResourceStr)); + item.put("ecmInstance", latestRecord.getEcmInstance()); + String engineType = latestRecord.getLabelValue().split(",")[1].split("-")[0]; + item.put("engineType", engineType); + if (StringUtils.isNotBlank(queueName)) { + Map usedResourceMap = + ECResourceInfoUtils.getStringToMap(usedResourceStr); + Map yarn = MapUtils.getMap(usedResourceMap, "yarn", new HashMap()); + String queueNameStr = String.valueOf(yarn.getOrDefault("queueName", "")); + if (StringUtils.isNotBlank(queueNameStr) && queueName.equals(queueNameStr)) { + resultList.add(item); + } + } else { resultList.add(item); } } catch (JsonProcessingException e) { From 2a222309aeb77df4bbb8a7d908127b941fb842e2 Mon Sep 17 00:00:00 2001 From: huangKai-2323 <62878639+huangKai-2323@users.noreply.github.com> Date: Wed, 26 Apr 2023 14:14:27 +0800 Subject: [PATCH 078/261] add doubleToStringTest --- .../storage/utils/StorageUtilsTest.scala | 32 +++++++++++++++++++ 1 file changed, 32 insertions(+) create mode 100644 linkis-commons/linkis-storage/src/test/scala/org/apache/linkis/storage/utils/StorageUtilsTest.scala diff --git a/linkis-commons/linkis-storage/src/test/scala/org/apache/linkis/storage/utils/StorageUtilsTest.scala b/linkis-commons/linkis-storage/src/test/scala/org/apache/linkis/storage/utils/StorageUtilsTest.scala new file mode 100644 index 0000000000..d75a102c4e --- /dev/null +++ b/linkis-commons/linkis-storage/src/test/scala/org/apache/linkis/storage/utils/StorageUtilsTest.scala @@ -0,0 +1,32 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.storage.utils + +import org.junit.jupiter.api.{Assertions, DisplayName, Test} + +class StorageUtilsTest { + + @Test + @DisplayName("doubleToStringTest") + def doubleToStringTest(): Unit = { + val str = StorageUtils.doubleToString(Double.NaN) + Assertions.assertNotNull(str) + + } + +} From 937ae5e4301d3111b6fa9793e953e5326606ef01 Mon Sep 17 00:00:00 2001 From: peacewong Date: Wed, 26 Apr 2023 15:36:48 +0800 Subject: [PATCH 079/261] fix Fixed session issue --- .../common/utils/ByteTimeUtilsTest.java | 44 +++++++++---------- .../AccessibleExecutorConfiguration.scala | 12 +++++ .../service/DefaultAccessibleService.scala | 41 ++++++++++++++++- .../heartbeat/AMHeartbeatService.scala | 1 + 4 files changed, 73 insertions(+), 25 deletions(-) diff --git a/linkis-commons/linkis-common/src/test/java/org/apache/linkis/common/utils/ByteTimeUtilsTest.java b/linkis-commons/linkis-common/src/test/java/org/apache/linkis/common/utils/ByteTimeUtilsTest.java index 0d520cc05e..7b90f053ca 100644 --- a/linkis-commons/linkis-common/src/test/java/org/apache/linkis/common/utils/ByteTimeUtilsTest.java +++ b/linkis-commons/linkis-common/src/test/java/org/apache/linkis/common/utils/ByteTimeUtilsTest.java @@ -17,35 +17,31 @@ package org.apache.linkis.common.utils; -import static org.junit.jupiter.api.Assertions.*; - -import org.apache.linkis.common.exception.LinkisSecurityException; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; -class ByteTimeUtilsTest { - @Test - void byteStringAsBytes() { - } +import static org.junit.jupiter.api.Assertions.*; + +class ByteTimeUtilsTest { - @Test - void byteStringAsKb() { - } + @Test + void byteStringAsBytes() {} - @Test - void byteStringAsMb() { - } + @Test + void byteStringAsKb() {} - @Test - void byteStringAsGb() { - Long res = ByteTimeUtils.byteStringAsGb("1G"); - Assertions.assertEquals(res,1); + @Test + void byteStringAsMb() {} - Assertions.assertThrows( - NullPointerException.class, - () -> { - ByteTimeUtils.byteStringAsGb("512"); - }); + @Test + void byteStringAsGb() { + Long res = ByteTimeUtils.byteStringAsGb("1G"); + Assertions.assertEquals(res, 1); - } -} \ No newline at end of file + Assertions.assertThrows( + NullPointerException.class, + () -> { + ByteTimeUtils.byteStringAsGb("512"); + }); + } +} diff --git a/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/org/apache/linkis/engineconn/acessible/executor/conf/AccessibleExecutorConfiguration.scala b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/org/apache/linkis/engineconn/acessible/executor/conf/AccessibleExecutorConfiguration.scala index 0eb211f731..26a25a1539 100644 --- a/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/org/apache/linkis/engineconn/acessible/executor/conf/AccessibleExecutorConfiguration.scala +++ b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/org/apache/linkis/engineconn/acessible/executor/conf/AccessibleExecutorConfiguration.scala @@ -55,4 +55,16 @@ object AccessibleExecutorConfiguration { val ENABLE_MAINTAIN_CREATORS = CommonVars("wds.linkis.engineconn.maintain.cretors", "IDE") + val REPORTING_DELAY_MS = CommonVars( + "linkis.engineconn.heartbeat.report.delay", + 20, + "Heartbeat status reporting delay, default 20ms, Negative numbers do not take effect" + ).getValue + + val REPORTING_IGNORE_MS = CommonVars( + "linkis.engineconn.heartbeat.report.ignore", + 3, + "Heartbeat status report repeated ignore, default 3ms,Negative numbers do not take effect" + ).getValue + } diff --git a/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/org/apache/linkis/engineconn/acessible/executor/service/DefaultAccessibleService.scala b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/org/apache/linkis/engineconn/acessible/executor/service/DefaultAccessibleService.scala index d07d16ce27..157c508e17 100644 --- a/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/org/apache/linkis/engineconn/acessible/executor/service/DefaultAccessibleService.scala +++ b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/org/apache/linkis/engineconn/acessible/executor/service/DefaultAccessibleService.scala @@ -19,6 +19,7 @@ package org.apache.linkis.engineconn.acessible.executor.service import org.apache.linkis.DataWorkCloudApplication import org.apache.linkis.common.utils.{Logging, Utils} +import org.apache.linkis.engineconn.acessible.executor.conf.AccessibleExecutorConfiguration import org.apache.linkis.engineconn.acessible.executor.entity.AccessibleExecutor import org.apache.linkis.engineconn.acessible.executor.listener.event.{ ExecutorCompletedEvent, @@ -57,6 +58,12 @@ class DefaultAccessibleService extends AccessibleService with Logging { private var shutDownHooked: Boolean = false + private var lastStatusChanged: Long = System.currentTimeMillis() + + private var lastStatus: NodeStatus = null + + private var lastThreadName: String = null + @Receiver override def dealEngineStopRequest( engineSuicideRequest: EngineSuicideRequest, @@ -167,7 +174,39 @@ class DefaultAccessibleService extends AccessibleService with Logging { override def onExecutorStatusChanged( executorStatusChangedEvent: ExecutorStatusChangedEvent ): Unit = { - reportHeartBeatMsg(executorStatusChangedEvent.executor) + val sinceLastTime = System.currentTimeMillis() - lastStatusChanged + val reportDelay = AccessibleExecutorConfiguration.REPORTING_DELAY_MS + if ( + reportDelay > 0 && executorStatusChangedEvent.toStatus != lastStatus && reportDelay > sinceLastTime + ) { + logger.info( + "In order to ensure that the previous state is consumed first, sleep here {} s", + reportDelay * 2 + ) + + Thread.sleep(reportDelay * 2) + } + val ignoreTime = AccessibleExecutorConfiguration.REPORTING_IGNORE_MS + val currentThreadName = Thread.currentThread().getName + if ( + ignoreTime > 0 && executorStatusChangedEvent.toStatus == lastStatus && ignoreTime > sinceLastTime && currentThreadName + .equals(lastThreadName) + ) { + logger.info( + "If the status is the same and the time is short and the thread is the same, no status report is performed {}", + executorStatusChangedEvent + ) + } else if ( + NodeStatus.Busy == lastStatus && executorStatusChangedEvent.toStatus == NodeStatus.Idle + ) { + logger.info("The state transition from Busy to Idle is not reported") + } else { + reportHeartBeatMsg(executorStatusChangedEvent.executor) + } + logger.info("Finished to report status {}", executorStatusChangedEvent) + lastStatusChanged = System.currentTimeMillis() + lastStatus = executorStatusChangedEvent.toStatus + lastThreadName = currentThreadName } private def reportHeartBeatMsg(executor: Executor): Unit = { diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/service/heartbeat/AMHeartbeatService.scala b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/service/heartbeat/AMHeartbeatService.scala index 25db0b8303..e3381985d0 100644 --- a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/service/heartbeat/AMHeartbeatService.scala +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/service/heartbeat/AMHeartbeatService.scala @@ -78,6 +78,7 @@ class AMHeartbeatService extends HeartbeatService with Logging { nodeMetrics.setStatus(0) } nodeMetricManagerPersistence.addOrupdateNodeMetrics(nodeMetrics) + logger.info(s"Finished to deal nodeHeartbeatMsg $nodeHeartbeatMsg") } } From 2e3ff3951d6896bf697ee4ba87e3d5e26c63c996 Mon Sep 17 00:00:00 2001 From: peacewong Date: Wed, 26 Apr 2023 15:52:07 +0800 Subject: [PATCH 080/261] update to ms --- .../acessible/executor/service/DefaultAccessibleService.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/org/apache/linkis/engineconn/acessible/executor/service/DefaultAccessibleService.scala b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/org/apache/linkis/engineconn/acessible/executor/service/DefaultAccessibleService.scala index 157c508e17..06fd13b0e9 100644 --- a/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/org/apache/linkis/engineconn/acessible/executor/service/DefaultAccessibleService.scala +++ b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/org/apache/linkis/engineconn/acessible/executor/service/DefaultAccessibleService.scala @@ -180,7 +180,7 @@ class DefaultAccessibleService extends AccessibleService with Logging { reportDelay > 0 && executorStatusChangedEvent.toStatus != lastStatus && reportDelay > sinceLastTime ) { logger.info( - "In order to ensure that the previous state is consumed first, sleep here {} s", + "In order to ensure that the previous state is consumed first, sleep here {} ms", reportDelay * 2 ) From db5026f78c817f37fd5f0e2b863688ff6d0ec2a8 Mon Sep 17 00:00:00 2001 From: huangKai-2323 <62878639+huangKai-2323@users.noreply.github.com> Date: Wed, 26 Apr 2023 16:26:51 +0800 Subject: [PATCH 081/261] format code --- .../am/service/impl/ECResourceInfoServiceImpl.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/service/impl/ECResourceInfoServiceImpl.java b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/service/impl/ECResourceInfoServiceImpl.java index 3e8bc0c441..b55d7c2630 100644 --- a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/service/impl/ECResourceInfoServiceImpl.java +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/service/impl/ECResourceInfoServiceImpl.java @@ -164,11 +164,11 @@ public List> getECResourceInfoList( HashMap heartbeatMap = BDPJettyServerHelper.gson() .fromJson(ecNodeinfo.getHeartbeatMsg(), new HashMap<>().getClass()); - Object lastUnlockTimeMills1 = + Object lastUnlockTimestampObject = Optional.ofNullable(heartbeatMap.get("lastUnlockTimestamp")).orElse(0); - BigDecimal lastUnlockTimeMills = - new BigDecimal(String.valueOf(lastUnlockTimeMills1)); - lastUnlockTimestamp = lastUnlockTimeMills.longValue(); + BigDecimal lastUnlockTimestampBigDecimal = + new BigDecimal(String.valueOf(lastUnlockTimestampObject)); + lastUnlockTimestamp = lastUnlockTimestampBigDecimal.longValue(); } item.put("lastUnlockTimestamp", lastUnlockTimestamp); item.put("useResource", ECResourceInfoUtils.getStringToMap(usedResourceStr)); From 1325cc0699fdb8376d5b87da1b39dc9210b8d72c Mon Sep 17 00:00:00 2001 From: huangKai-2323 <62878639+huangKai-2323@users.noreply.github.com> Date: Wed, 26 Apr 2023 16:29:04 +0800 Subject: [PATCH 082/261] format code --- .../org/apache/linkis/storage/utils/StorageUtilsTest.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/linkis-commons/linkis-storage/src/test/scala/org/apache/linkis/storage/utils/StorageUtilsTest.scala b/linkis-commons/linkis-storage/src/test/scala/org/apache/linkis/storage/utils/StorageUtilsTest.scala index d75a102c4e..6ae15782ee 100644 --- a/linkis-commons/linkis-storage/src/test/scala/org/apache/linkis/storage/utils/StorageUtilsTest.scala +++ b/linkis-commons/linkis-storage/src/test/scala/org/apache/linkis/storage/utils/StorageUtilsTest.scala @@ -25,7 +25,7 @@ class StorageUtilsTest { @DisplayName("doubleToStringTest") def doubleToStringTest(): Unit = { val str = StorageUtils.doubleToString(Double.NaN) - Assertions.assertNotNull(str) + Assertions.assertEquals("NaN", str) } From ff982b29e9e304e56148c612a4f08b606eeb51cc Mon Sep 17 00:00:00 2001 From: huangKai-2323 <62878639+huangKai-2323@users.noreply.github.com> Date: Wed, 26 Apr 2023 20:09:34 +0800 Subject: [PATCH 083/261] [1.1.11] Optimize UDF function logic (#131) * Code optimization * add udf function createtime && updatetime column * add udf updatetime sql --- linkis-dist/package/db/linkis_ddl.sql | 11 +- .../upgrade/1.4.0_schema/mysql/linkis_ddl.sql | 31 +++ .../server/conf/UdfTreeConf.java | 26 +++ .../server/dao/UdfBaseInfoMapper.java | 29 +++ .../server/domain/UdfBaseInfoEntity.java | 205 ++++++++++++++++++ .../server/domain/UdfManagerEntity.java | 23 ++ .../server/restful/UdfManagerRestfulApi.java | 23 +- .../server/restful/UdfTreeRestfulApi.java | 60 ++++- .../server/service/UdfBaseInfoService.java | 28 +++ .../service/impl/UdfBaseInfoServicelmpl.java | 30 +++ .../mapper/common/UdfBaseInfoMapper.xml | 24 ++ .../src/test/resources/create.sql | 18 +- .../src/test/resources/data.sql | 7 + .../apache/linkis/udf/entity/UDFManager.java | 20 ++ .../main/resources/mapper/common/UDFDao.xml | 4 +- .../resources/mapper/common/UDFVersionDao.xml | 9 +- 16 files changed, 532 insertions(+), 16 deletions(-) create mode 100644 linkis-dist/package/db/upgrade/1.4.0_schema/mysql/linkis_ddl.sql create mode 100644 linkis-public-enhancements/linkis-basedata-manager/src/main/java/org/apache/linkis/basedatamanager/server/conf/UdfTreeConf.java create mode 100644 linkis-public-enhancements/linkis-basedata-manager/src/main/java/org/apache/linkis/basedatamanager/server/dao/UdfBaseInfoMapper.java create mode 100644 linkis-public-enhancements/linkis-basedata-manager/src/main/java/org/apache/linkis/basedatamanager/server/domain/UdfBaseInfoEntity.java create mode 100644 linkis-public-enhancements/linkis-basedata-manager/src/main/java/org/apache/linkis/basedatamanager/server/service/UdfBaseInfoService.java create mode 100644 linkis-public-enhancements/linkis-basedata-manager/src/main/java/org/apache/linkis/basedatamanager/server/service/impl/UdfBaseInfoServicelmpl.java create mode 100644 linkis-public-enhancements/linkis-basedata-manager/src/main/resources/mapper/common/UdfBaseInfoMapper.xml diff --git a/linkis-dist/package/db/linkis_ddl.sql b/linkis-dist/package/db/linkis_ddl.sql index 8fe51ceef8..10e1ada0a6 100644 --- a/linkis-dist/package/db/linkis_ddl.sql +++ b/linkis-dist/package/db/linkis_ddl.sql @@ -153,6 +153,8 @@ DROP TABLE IF EXISTS `linkis_ps_udf_manager`; CREATE TABLE `linkis_ps_udf_manager` ( `id` bigint(20) NOT NULL AUTO_INCREMENT, `user_name` varchar(20) DEFAULT NULL, + `update_time` datetime DEFAULT CURRENT_TIMESTAMP, + `create_time` datetime DEFAULT CURRENT_TIMESTAMP, PRIMARY KEY (`id`) ) ENGINE=InnoDB DEFAULT CHARSET=utf8; @@ -166,6 +168,8 @@ CREATE TABLE `linkis_ps_udf_shared_group` ( `id` bigint(20) NOT NULL AUTO_INCREMENT, `udf_id` bigint(20) NOT NULL, `shared_group` varchar(50) NOT NULL, + `update_time` datetime DEFAULT CURRENT_TIMESTAMP, + `create_time` datetime DEFAULT CURRENT_TIMESTAMP, PRIMARY KEY (`id`) ) ENGINE=InnoDB DEFAULT CHARSET=utf8; @@ -174,7 +178,9 @@ CREATE TABLE `linkis_ps_udf_shared_info` ( `id` bigint(20) PRIMARY KEY NOT NULL AUTO_INCREMENT, `udf_id` bigint(20) NOT NULL, - `user_name` varchar(50) NOT NULL + `user_name` varchar(50) NOT NULL, + `update_time` datetime DEFAULT CURRENT_TIMESTAMP, + `create_time` datetime DEFAULT CURRENT_TIMESTAMP ) ENGINE=InnoDB DEFAULT CHARSET=utf8; -- ---------------------------- @@ -203,6 +209,8 @@ CREATE TABLE `linkis_ps_udf_user_load` ( `id` bigint(20) NOT NULL AUTO_INCREMENT, `udf_id` bigint(20) NOT NULL, `user_name` varchar(50) NOT NULL, + `update_time` datetime DEFAULT CURRENT_TIMESTAMP, + `create_time` datetime DEFAULT CURRENT_TIMESTAMP, PRIMARY KEY (`id`) ) ENGINE=InnoDB DEFAULT CHARSET=utf8; @@ -235,6 +243,7 @@ CREATE TABLE `linkis_ps_udf_version` ( `use_format` varchar(255) DEFAULT NULL, `description` varchar(255) NOT NULL COMMENT 'version desc', `create_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + `update_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, `md5` varchar(100) DEFAULT NULL, PRIMARY KEY (`id`) ) ENGINE=InnoDB DEFAULT CHARSET=utf8; diff --git a/linkis-dist/package/db/upgrade/1.4.0_schema/mysql/linkis_ddl.sql b/linkis-dist/package/db/upgrade/1.4.0_schema/mysql/linkis_ddl.sql new file mode 100644 index 0000000000..f431faee65 --- /dev/null +++ b/linkis-dist/package/db/upgrade/1.4.0_schema/mysql/linkis_ddl.sql @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +ALTER TABLE `linkis_ps_udf_manager` ADD COLUMN `create_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'create time'; +ALTER TABLE `linkis_ps_udf_manager` ADD COLUMN `update_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'update_time'; + +ALTER TABLE `linkis_ps_udf_shared_group` ADD COLUMN `create_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'create time'; +ALTER TABLE `linkis_ps_udf_shared_group` ADD COLUMN `update_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'update_time'; + +ALTER TABLE `linkis_ps_udf_shared_info` ADD COLUMN `create_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'create time'; +ALTER TABLE `linkis_ps_udf_shared_info` ADD COLUMN `update_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'update_time'; + +ALTER TABLE `linkis_ps_udf_user_load` ADD COLUMN `create_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'create time'; +ALTER TABLE `linkis_ps_udf_user_load` ADD COLUMN `update_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'update_time'; + + +ALTER TABLE `linkis_ps_udf_version` ADD COLUMN `update_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'update_time'; diff --git a/linkis-public-enhancements/linkis-basedata-manager/src/main/java/org/apache/linkis/basedatamanager/server/conf/UdfTreeConf.java b/linkis-public-enhancements/linkis-basedata-manager/src/main/java/org/apache/linkis/basedatamanager/server/conf/UdfTreeConf.java new file mode 100644 index 0000000000..ded1d7e434 --- /dev/null +++ b/linkis-public-enhancements/linkis-basedata-manager/src/main/java/org/apache/linkis/basedatamanager/server/conf/UdfTreeConf.java @@ -0,0 +1,26 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.basedatamanager.server.conf; + +import org.apache.linkis.common.conf.CommonVars; + +public class UdfTreeConf { + + public static final CommonVars UDF_FUN_SYSTEM_CATEGORY = + CommonVars.apply("linkis.udf.fun.system.category", "user_name,sys,expire,share,bdp"); +} diff --git a/linkis-public-enhancements/linkis-basedata-manager/src/main/java/org/apache/linkis/basedatamanager/server/dao/UdfBaseInfoMapper.java b/linkis-public-enhancements/linkis-basedata-manager/src/main/java/org/apache/linkis/basedatamanager/server/dao/UdfBaseInfoMapper.java new file mode 100644 index 0000000000..c2d585993b --- /dev/null +++ b/linkis-public-enhancements/linkis-basedata-manager/src/main/java/org/apache/linkis/basedatamanager/server/dao/UdfBaseInfoMapper.java @@ -0,0 +1,29 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.basedatamanager.server.dao; + +import org.apache.linkis.basedatamanager.server.domain.UdfBaseInfoEntity; + +import com.baomidou.mybatisplus.core.mapper.BaseMapper; + +/** + * @description Database operation Mapper for the linkis_PS_UDF_tree table + * @createDate 2022-08-13 15:13:27 @Entity + * org.apache.linkis.basedatamanager.server.domain.LinkisPsUdfTree + */ +public interface UdfBaseInfoMapper extends BaseMapper {} diff --git a/linkis-public-enhancements/linkis-basedata-manager/src/main/java/org/apache/linkis/basedatamanager/server/domain/UdfBaseInfoEntity.java b/linkis-public-enhancements/linkis-basedata-manager/src/main/java/org/apache/linkis/basedatamanager/server/domain/UdfBaseInfoEntity.java new file mode 100644 index 0000000000..0c8776ed26 --- /dev/null +++ b/linkis-public-enhancements/linkis-basedata-manager/src/main/java/org/apache/linkis/basedatamanager/server/domain/UdfBaseInfoEntity.java @@ -0,0 +1,205 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.basedatamanager.server.domain; + +import java.io.Serializable; +import java.util.Date; +import java.util.Objects; + +import com.baomidou.mybatisplus.annotation.IdType; +import com.baomidou.mybatisplus.annotation.TableField; +import com.baomidou.mybatisplus.annotation.TableId; +import com.baomidou.mybatisplus.annotation.TableName; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; + +/** @TableName linkis_ps_udf_baseinfo */ +@TableName(value = "linkis_ps_udf_baseinfo") +@JsonIgnoreProperties(ignoreUnknown = true) +public class UdfBaseInfoEntity implements Serializable { + + @TableId(type = IdType.AUTO) + private Long id; + + private String createUser; + private String udfName; + private Integer udfType; + private Boolean isExpire; + private Boolean isShared; + private Long treeId; + private Date createTime; + private Date updateTime; + private String sys; + private String clusterName; + + @TableField(exist = false) + private static final long serialVersionUID = 1L; + + public Long getId() { + return id; + } + + public void setId(Long id) { + this.id = id; + } + + public String getCreateUser() { + return createUser; + } + + public void setCreateUser(String createUser) { + this.createUser = createUser; + } + + public String getUdfName() { + return udfName; + } + + public void setUdfName(String udfName) { + this.udfName = udfName; + } + + public Integer getUdfType() { + return udfType; + } + + public void setUdfType(Integer udfType) { + this.udfType = udfType; + } + + public Boolean getExpire() { + return isExpire; + } + + public void setExpire(Boolean expire) { + isExpire = expire; + } + + public Boolean getShared() { + return isShared; + } + + public void setShared(Boolean shared) { + isShared = shared; + } + + public Long getTreeId() { + return treeId; + } + + public void setTreeId(Long treeId) { + this.treeId = treeId; + } + + public Date getCreateTime() { + return createTime; + } + + public void setCreateTime(Date createTime) { + this.createTime = createTime; + } + + public Date getUpdateTime() { + return updateTime; + } + + public void setUpdateTime(Date updateTime) { + this.updateTime = updateTime; + } + + public String getSys() { + return sys; + } + + public void setSys(String sys) { + this.sys = sys; + } + + public String getClusterName() { + return clusterName; + } + + public void setClusterName(String clusterName) { + this.clusterName = clusterName; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + UdfBaseInfoEntity that = (UdfBaseInfoEntity) o; + return Objects.equals(id, that.id) + && Objects.equals(createUser, that.createUser) + && Objects.equals(udfName, that.udfName) + && Objects.equals(udfType, that.udfType) + && Objects.equals(isExpire, that.isExpire) + && Objects.equals(isShared, that.isShared) + && Objects.equals(treeId, that.treeId) + && Objects.equals(createTime, that.createTime) + && Objects.equals(updateTime, that.updateTime) + && Objects.equals(sys, that.sys) + && Objects.equals(clusterName, that.clusterName); + } + + @Override + public int hashCode() { + return Objects.hash( + id, + createUser, + udfName, + udfType, + isExpire, + isShared, + treeId, + createTime, + updateTime, + sys, + clusterName); + } + + @Override + public String toString() { + return "UdfBaseInfoEntity{" + + "id=" + + id + + ", createUser='" + + createUser + + '\'' + + ", udfName='" + + udfName + + '\'' + + ", udfType=" + + udfType + + ", isExpire=" + + isExpire + + ", isShared=" + + isShared + + ", treeId=" + + treeId + + ", createTime=" + + createTime + + ", updateTime=" + + updateTime + + ", sys='" + + sys + + '\'' + + ", clusterName='" + + clusterName + + '\'' + + '}'; + } +} diff --git a/linkis-public-enhancements/linkis-basedata-manager/src/main/java/org/apache/linkis/basedatamanager/server/domain/UdfManagerEntity.java b/linkis-public-enhancements/linkis-basedata-manager/src/main/java/org/apache/linkis/basedatamanager/server/domain/UdfManagerEntity.java index fa6db837e1..8c5abbeec1 100644 --- a/linkis-public-enhancements/linkis-basedata-manager/src/main/java/org/apache/linkis/basedatamanager/server/domain/UdfManagerEntity.java +++ b/linkis-public-enhancements/linkis-basedata-manager/src/main/java/org/apache/linkis/basedatamanager/server/domain/UdfManagerEntity.java @@ -18,6 +18,7 @@ package org.apache.linkis.basedatamanager.server.domain; import java.io.Serializable; +import java.util.Date; import com.baomidou.mybatisplus.annotation.IdType; import com.baomidou.mybatisplus.annotation.TableField; @@ -36,6 +37,12 @@ public class UdfManagerEntity implements Serializable { /** */ private String userName; + /** */ + private Date createTime; + + /** */ + private Date updateTime; + @TableField(exist = false) private static final long serialVersionUID = 1L; @@ -59,6 +66,22 @@ public void setUserName(String userName) { this.userName = userName; } + public Date getCreateTime() { + return createTime; + } + + public void setCreateTime(Date createTime) { + this.createTime = createTime; + } + + public Date getUpdateTime() { + return updateTime; + } + + public void setUpdateTime(Date updateTime) { + this.updateTime = updateTime; + } + @Override public boolean equals(Object that) { if (this == that) { diff --git a/linkis-public-enhancements/linkis-basedata-manager/src/main/java/org/apache/linkis/basedatamanager/server/restful/UdfManagerRestfulApi.java b/linkis-public-enhancements/linkis-basedata-manager/src/main/java/org/apache/linkis/basedatamanager/server/restful/UdfManagerRestfulApi.java index f6e684f837..f048b12f4b 100644 --- a/linkis-public-enhancements/linkis-basedata-manager/src/main/java/org/apache/linkis/basedatamanager/server/restful/UdfManagerRestfulApi.java +++ b/linkis-public-enhancements/linkis-basedata-manager/src/main/java/org/apache/linkis/basedatamanager/server/restful/UdfManagerRestfulApi.java @@ -32,6 +32,8 @@ import javax.servlet.http.HttpServletRequest; +import java.util.Date; + import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper; import com.github.pagehelper.PageInfo; import io.swagger.annotations.Api; @@ -89,6 +91,8 @@ public Message add(HttpServletRequest request, @RequestBody UdfManagerEntity udf new QueryWrapper<>(udfManagerEntity).eq("user_name", udfManagerEntity.getUserName()); UdfManagerEntity udfManager = udfManagerService.getOne(queryWrapper); if (udfManager == null) { + udfManagerEntity.setCreateTime(new Date()); + udfManagerEntity.setUpdateTime(new Date()); boolean result = udfManagerService.save(udfManagerEntity); return Message.ok("").data("result", result); } else { @@ -103,7 +107,12 @@ public Message add(HttpServletRequest request, @RequestBody UdfManagerEntity udf httpMethod = "DELETE") @RequestMapping(path = "/{id}", method = RequestMethod.DELETE) public Message remove(HttpServletRequest request, @PathVariable("id") Long id) { - ModuleUserUtils.getOperationUser(request, "Remove a UDF Manager Record,id:" + id.toString()); + String username = + ModuleUserUtils.getOperationUser( + request, "Remove a UDF Manager Record,id:" + id.toString()); + if (!Configuration.isAdmin(username)) { + return Message.error("User '" + username + "' is not admin user[非管理员用户]"); + } boolean result = udfManagerService.removeById(id); return Message.ok("").data("result", result); } @@ -121,7 +130,15 @@ public Message update( if (!Configuration.isAdmin(username)) { return Message.error("User '" + username + "' is not admin user[非管理员用户]"); } - boolean result = udfManagerService.updateById(udfManagerEntity); - return Message.ok("").data("result", result); + QueryWrapper queryWrapper = + new QueryWrapper<>(udfManagerEntity).eq("user_name", udfManagerEntity.getUserName()); + UdfManagerEntity udfManager = udfManagerService.getOne(queryWrapper); + if (udfManager == null) { + udfManager.setUpdateTime(new Date()); + boolean result = udfManagerService.updateById(udfManagerEntity); + return Message.ok("").data("result", result); + } else { + return Message.error("The username already exists,Please update again!"); + } } } diff --git a/linkis-public-enhancements/linkis-basedata-manager/src/main/java/org/apache/linkis/basedatamanager/server/restful/UdfTreeRestfulApi.java b/linkis-public-enhancements/linkis-basedata-manager/src/main/java/org/apache/linkis/basedatamanager/server/restful/UdfTreeRestfulApi.java index e5c9cbc442..c9fd2bf985 100644 --- a/linkis-public-enhancements/linkis-basedata-manager/src/main/java/org/apache/linkis/basedatamanager/server/restful/UdfTreeRestfulApi.java +++ b/linkis-public-enhancements/linkis-basedata-manager/src/main/java/org/apache/linkis/basedatamanager/server/restful/UdfTreeRestfulApi.java @@ -17,12 +17,18 @@ package org.apache.linkis.basedatamanager.server.restful; +import org.apache.linkis.basedatamanager.server.conf.UdfTreeConf; +import org.apache.linkis.basedatamanager.server.domain.UdfBaseInfoEntity; import org.apache.linkis.basedatamanager.server.domain.UdfTreeEntity; +import org.apache.linkis.basedatamanager.server.service.UdfBaseInfoService; import org.apache.linkis.basedatamanager.server.service.UdfTreeService; import org.apache.linkis.common.conf.Configuration; import org.apache.linkis.server.Message; import org.apache.linkis.server.utils.ModuleUserUtils; +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.lang3.StringUtils; + import org.springframework.beans.factory.annotation.Autowired; import org.springframework.web.bind.annotation.PathVariable; import org.springframework.web.bind.annotation.RequestBody; @@ -32,8 +38,11 @@ import javax.servlet.http.HttpServletRequest; +import java.util.ArrayList; +import java.util.Arrays; import java.util.List; +import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper; import com.github.pagehelper.PageInfo; import io.swagger.annotations.Api; import io.swagger.annotations.ApiImplicitParam; @@ -47,6 +56,8 @@ public class UdfTreeRestfulApi { @Autowired UdfTreeService udfTreeService; + @Autowired UdfBaseInfoService udfBaseinfoService; + @ApiImplicitParams({ @ApiImplicitParam(paramType = "query", dataType = "string", name = "searchName"), @ApiImplicitParam(paramType = "query", dataType = "int", name = "currentPage"), @@ -62,12 +73,33 @@ public Message list( return Message.ok("").data("list", pageList); } + @ApiImplicitParams({ + @ApiImplicitParam(paramType = "query", dataType = "string", name = "searchName"), + @ApiImplicitParam(paramType = "query", dataType = "string", name = "category") + }) @ApiOperation(value = "all", notes = "Query all data of UDF Tree", httpMethod = "GET") @RequestMapping(path = "/all", method = RequestMethod.GET) - public Message all(HttpServletRequest request, String searchName) { + public Message all(HttpServletRequest request, String searchName, String category) { ModuleUserUtils.getOperationUser( request, "Query all data of UDF Tree,search name:" + searchName); - List udfTreeEntityList = udfTreeService.list(); + List udfTreeEntityList = new ArrayList<>(); + UdfTreeEntity entity = new UdfTreeEntity(); + entity.setCategory(category); + QueryWrapper querySysWrapper = + new QueryWrapper<>(entity) + .eq("category", entity.getCategory()) + .in( + "user_name", + Arrays.asList(UdfTreeConf.UDF_FUN_SYSTEM_CATEGORY.getValue().split(","))); + udfTreeEntityList = udfTreeService.list(querySysWrapper); + if (StringUtils.isNotBlank(searchName) && StringUtils.isNotBlank(category)) { + entity.setUserName(searchName); + QueryWrapper queryWrapper = + new QueryWrapper<>(entity) + .eq("user_name", entity.getUserName()) + .eq("category", entity.getCategory()); + udfTreeEntityList.addAll(udfTreeService.list(queryWrapper)); + } return Message.ok("").data("list", udfTreeEntityList); } @@ -100,9 +132,27 @@ public Message add(HttpServletRequest request, @RequestBody UdfTreeEntity udfTre @ApiOperation(value = "remove", notes = "Remove a UDF Tree Record by id", httpMethod = "DELETE") @RequestMapping(path = "/{id}", method = RequestMethod.DELETE) public Message remove(HttpServletRequest request, @PathVariable("id") Long id) { - ModuleUserUtils.getOperationUser(request, "Remove a UDF Tree Record,id:" + id.toString()); - boolean result = udfTreeService.removeById(id); - return Message.ok("").data("result", result); + String username = + ModuleUserUtils.getOperationUser(request, "Remove a UDF Tree Record,id:" + id.toString()); + if (!Configuration.isAdmin(username)) { + return Message.error("User '" + username + "' is not admin user[非管理员用户]"); + } + UdfTreeEntity entity = udfTreeService.getById(id); + if (null != entity && entity.getParent() == -1) { + return Message.error("The root directory is forbidden to delete[\"根目录禁止删除\"]"); + } + QueryWrapper queryWrapper = + new QueryWrapper<>(new UdfTreeEntity()).eq("parent", id); + List folderList = udfTreeService.list(queryWrapper); + QueryWrapper udfQueryWrapper = + new QueryWrapper<>(new UdfBaseInfoEntity()).eq("tree_id", id); + List functoinList = udfBaseinfoService.list(udfQueryWrapper); + if (CollectionUtils.isEmpty(folderList) && CollectionUtils.isEmpty(functoinList)) { + boolean result = udfTreeService.removeById(id); + return Message.ok("").data("result", result); + } else { + return Message.error("Please delete the subdirectory first[请先删除子目录]"); + } } @ApiImplicitParams({ diff --git a/linkis-public-enhancements/linkis-basedata-manager/src/main/java/org/apache/linkis/basedatamanager/server/service/UdfBaseInfoService.java b/linkis-public-enhancements/linkis-basedata-manager/src/main/java/org/apache/linkis/basedatamanager/server/service/UdfBaseInfoService.java new file mode 100644 index 0000000000..032c818b43 --- /dev/null +++ b/linkis-public-enhancements/linkis-basedata-manager/src/main/java/org/apache/linkis/basedatamanager/server/service/UdfBaseInfoService.java @@ -0,0 +1,28 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.basedatamanager.server.service; + +import org.apache.linkis.basedatamanager.server.domain.UdfBaseInfoEntity; + +import com.baomidou.mybatisplus.extension.service.IService; + +/** + * @description Database operation Service for the [linkis_ps_udf_baseinfo] table + * @createDate 2022-08-13 15:13:27 + */ +public interface UdfBaseInfoService extends IService {} diff --git a/linkis-public-enhancements/linkis-basedata-manager/src/main/java/org/apache/linkis/basedatamanager/server/service/impl/UdfBaseInfoServicelmpl.java b/linkis-public-enhancements/linkis-basedata-manager/src/main/java/org/apache/linkis/basedatamanager/server/service/impl/UdfBaseInfoServicelmpl.java new file mode 100644 index 0000000000..c3c62854a1 --- /dev/null +++ b/linkis-public-enhancements/linkis-basedata-manager/src/main/java/org/apache/linkis/basedatamanager/server/service/impl/UdfBaseInfoServicelmpl.java @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.basedatamanager.server.service.impl; + +import org.apache.linkis.basedatamanager.server.dao.UdfBaseInfoMapper; +import org.apache.linkis.basedatamanager.server.domain.UdfBaseInfoEntity; +import org.apache.linkis.basedatamanager.server.service.UdfBaseInfoService; + +import org.springframework.stereotype.Service; + +import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl; + +@Service +public class UdfBaseInfoServicelmpl extends ServiceImpl + implements UdfBaseInfoService {} diff --git a/linkis-public-enhancements/linkis-basedata-manager/src/main/resources/mapper/common/UdfBaseInfoMapper.xml b/linkis-public-enhancements/linkis-basedata-manager/src/main/resources/mapper/common/UdfBaseInfoMapper.xml new file mode 100644 index 0000000000..30069a71f5 --- /dev/null +++ b/linkis-public-enhancements/linkis-basedata-manager/src/main/resources/mapper/common/UdfBaseInfoMapper.xml @@ -0,0 +1,24 @@ + + + + + + + diff --git a/linkis-public-enhancements/linkis-basedata-manager/src/test/resources/create.sql b/linkis-public-enhancements/linkis-basedata-manager/src/test/resources/create.sql index 88c4ab9a2a..669be86918 100644 --- a/linkis-public-enhancements/linkis-basedata-manager/src/test/resources/create.sql +++ b/linkis-public-enhancements/linkis-basedata-manager/src/test/resources/create.sql @@ -231,4 +231,20 @@ CREATE TABLE `linkis_cg_engine_conn_plugin_bml_resources` `create_time` datetime NOT NULL COMMENT 'created time', `last_update_time` datetime NOT NULL COMMENT 'updated time', PRIMARY KEY (`id`) -); \ No newline at end of file +); + +DROP TABLE IF EXISTS `linkis_ps_udf_baseinfo`; +CREATE TABLE `linkis_ps_udf_baseinfo` ( + `id` bigint(20) NOT NULL AUTO_INCREMENT, + `create_user` varchar(50) NOT NULL, + `udf_name` varchar(255) NOT NULL, + `udf_type` int(11) DEFAULT '0', + `tree_id` bigint(20) NOT NULL, + `create_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + `update_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + `sys` varchar(255) NOT NULL DEFAULT 'ide' COMMENT 'source system', + `cluster_name` varchar(255) NOT NULL, + `is_expire` bit(1) DEFAULT NULL, + `is_shared` bit(1) DEFAULT NULL, + PRIMARY KEY (`id`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8; \ No newline at end of file diff --git a/linkis-public-enhancements/linkis-basedata-manager/src/test/resources/data.sql b/linkis-public-enhancements/linkis-basedata-manager/src/test/resources/data.sql index 680c530762..ed3b9f57a5 100644 --- a/linkis-public-enhancements/linkis-basedata-manager/src/test/resources/data.sql +++ b/linkis-public-enhancements/linkis-basedata-manager/src/test/resources/data.sql @@ -214,3 +214,10 @@ INSERT INTO `linkis_ps_configuration_key_engine_relation` (`id`, `config_key_id` INSERT INTO `linkis_ps_configuration_key_engine_relation` (`id`, `config_key_id`, `engine_type_label_id`) VALUES (51, 48, 11); INSERT INTO `linkis_ps_configuration_key_engine_relation` (`id`, `config_key_id`, `engine_type_label_id`) VALUES (52, 49, 11); INSERT INTO `linkis_ps_configuration_key_engine_relation` (`id`, `config_key_id`, `engine_type_label_id`) VALUES (53, 50, 11); + +DELETE FROM linkis_ps_udf_baseinfo; +INSERT INTO linkis_ps_udf_baseinfo (create_user,udf_name,udf_type,tree_id,create_time,update_time,sys,cluster_name,is_expire,is_shared) VALUES + ('hadoop','pyUdfTest',1,14,'2022-09-08 11:43:20','2022-09-08 11:43:20','IDE','all',NULL,NULL), + ('hadoop','jarUdf',0,14,'2022-09-08 14:53:56','2022-09-08 14:53:56','IDE','all',NULL,NULL), + ('hadoop','test',3,13,'2022-09-08 14:54:30','2022-09-08 14:54:30','IDE','all',NULL,NULL), + ('hadoop','scalaUdf1',4,13,'2022-09-08 14:55:57','2022-09-08 14:55:57','IDE','all',NULL,NULL); diff --git a/linkis-public-enhancements/linkis-udf/linkis-udf-common/src/main/java/org/apache/linkis/udf/entity/UDFManager.java b/linkis-public-enhancements/linkis-udf/linkis-udf-common/src/main/java/org/apache/linkis/udf/entity/UDFManager.java index 6859fdb91b..0591313ae8 100644 --- a/linkis-public-enhancements/linkis-udf/linkis-udf-common/src/main/java/org/apache/linkis/udf/entity/UDFManager.java +++ b/linkis-public-enhancements/linkis-udf/linkis-udf-common/src/main/java/org/apache/linkis/udf/entity/UDFManager.java @@ -17,9 +17,13 @@ package org.apache.linkis.udf.entity; +import java.util.Date; + public class UDFManager { private Integer id; private String userName; + private Date createTime; + private Date updateTime; public Integer getId() { return id; @@ -36,4 +40,20 @@ public String getUserName() { public void setUserName(String userName) { this.userName = userName; } + + public Date getCreateTime() { + return createTime; + } + + public void setCreateTime(Date createTime) { + this.createTime = createTime; + } + + public Date getUpdateTime() { + return updateTime; + } + + public void setUpdateTime(Date updateTime) { + this.updateTime = updateTime; + } } diff --git a/linkis-public-enhancements/linkis-udf/linkis-udf-service/src/main/resources/mapper/common/UDFDao.xml b/linkis-public-enhancements/linkis-udf/linkis-udf-service/src/main/resources/mapper/common/UDFDao.xml index 17133345b6..c876e92342 100644 --- a/linkis-public-enhancements/linkis-udf/linkis-udf-service/src/main/resources/mapper/common/UDFDao.xml +++ b/linkis-public-enhancements/linkis-udf/linkis-udf-service/src/main/resources/mapper/common/UDFDao.xml @@ -102,7 +102,7 @@ - INSERT INTO linkis_ps_udf_user_load (`udf_id`,`user_name`) VALUES (#{arg0},#{arg1}) + INSERT INTO linkis_ps_udf_user_load (`udf_id`,`user_name`,`create_time`,`update_time`) VALUES (#{arg0},#{arg1},now(),now()) - update linkis_ps_udf_user_load set user_name=#{newUser} where udf_id=#{udfId} and user_name=#{oldUser} + update linkis_ps_udf_user_load set user_name=#{newUser},update_time =now() where udf_id=#{udfId} and user_name=#{oldUser} - update linkis_ps_udf_version set is_published=#{isPublished} where udf_id=#{udfId} and + update linkis_ps_udf_version set is_published=#{isPublished},update_time =now() where udf_id=#{udfId} and bml_resource_version=#{version} @@ -92,7 +92,7 @@ - update linkis_ps_udf_version set bml_resource_id=#{resourceId},path=replace(`path`,#{oldUser},#{newUser}) + update linkis_ps_udf_version set bml_resource_id=#{resourceId},path=replace(`path`,#{oldUser},#{newUser}),update_time=now() where udf_id=#{udfId} @@ -101,6 +101,7 @@ SET path=#{path,jdbcType=VARCHAR}, register_format=#{registerFormat,jdbcType=VARCHAR}, use_format=#{useFormat,jdbcType=VARCHAR}, + update_time=now(), description=#{description,jdbcType=VARCHAR} WHERE id = #{id,jdbcType=BIGINT} From 407a80032167f898b7f4acb56048f3ac9d2205df Mon Sep 17 00:00:00 2001 From: aiceflower Date: Wed, 26 Apr 2023 21:54:12 +0800 Subject: [PATCH 084/261] update get configuration logic (#140) * update get configuration logic * add annotation --------- Co-authored-by: aiceflower --- .../configuration/service/ConfigurationService.scala | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/linkis-public-enhancements/linkis-configuration/src/main/scala/org/apache/linkis/configuration/service/ConfigurationService.scala b/linkis-public-enhancements/linkis-configuration/src/main/scala/org/apache/linkis/configuration/service/ConfigurationService.scala index 1a0f714522..9b5f237021 100644 --- a/linkis-public-enhancements/linkis-configuration/src/main/scala/org/apache/linkis/configuration/service/ConfigurationService.scala +++ b/linkis-public-enhancements/linkis-configuration/src/main/scala/org/apache/linkis/configuration/service/ConfigurationService.scala @@ -259,6 +259,12 @@ class ConfigurationService extends Logging { combinedLabel.asInstanceOf[CombinedLabelImpl] } + /** + * Priority: configs > defaultConfigs + * @param configs + * @param defaultConfigs + * @return + */ def buildTreeResult( configs: util.List[ConfigKeyValue], defaultConfigs: util.List[ConfigKeyValue] = new util.ArrayList[ConfigKeyValue]() @@ -269,9 +275,8 @@ class ConfigurationService extends Logging { defaultConfig.setIsUserDefined(false) configs.asScala.foreach(config => { if (config.getKey != null && config.getKey.equals(defaultConfig.getKey)) { - if (StringUtils.isNotBlank(config.getConfigValue)) { - defaultConfig.setConfigValue(config.getConfigValue) - } + // configValue also needs to be replaced when the value is empty + defaultConfig.setConfigValue(config.getConfigValue) defaultConfig.setConfigLabelId(config.getConfigLabelId) defaultConfig.setValueId(config.getValueId) defaultConfig.setIsUserDefined(true) From ab570988a634638ad323426386cac2e63c370617 Mon Sep 17 00:00:00 2001 From: Casion Date: Thu, 27 Apr 2023 01:04:48 +0800 Subject: [PATCH 085/261] fix class no found (#144) --- .../executor/service/DefaultNodeHeartbeatMsgManager.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/service/DefaultNodeHeartbeatMsgManager.scala b/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/service/DefaultNodeHeartbeatMsgManager.scala index b27aad9d95..30045a0ba9 100644 --- a/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/service/DefaultNodeHeartbeatMsgManager.scala +++ b/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/service/DefaultNodeHeartbeatMsgManager.scala @@ -65,7 +65,7 @@ class DefaultNodeHeartbeatMsgManager extends NodeHeartbeatMsgManager with Loggin ComputationEngineConnMetrics.getTotalLockTimeMills(status).asInstanceOf[Object] ) msgMap.put( - ECConstants.EC_LAST_UNLOCK_TIME_MILLS, + ECConstants.EC_LAST_UNLOCK_TIMESTAMP, ComputationEngineConnMetrics.getLastUnlockTimestamp(status).asInstanceOf[Object] ) case _ => From 9e02cee4902a871122322e3e9592ee243682b48f Mon Sep 17 00:00:00 2001 From: peacewong Date: Thu, 27 Apr 2023 14:54:45 +0800 Subject: [PATCH 086/261] update test --- .../common/utils/ByteTimeUtilsTest.java | 152 ++++++++++++++++-- 1 file changed, 140 insertions(+), 12 deletions(-) diff --git a/linkis-commons/linkis-common/src/test/java/org/apache/linkis/common/utils/ByteTimeUtilsTest.java b/linkis-commons/linkis-common/src/test/java/org/apache/linkis/common/utils/ByteTimeUtilsTest.java index 7b90f053ca..f548d89d46 100644 --- a/linkis-commons/linkis-common/src/test/java/org/apache/linkis/common/utils/ByteTimeUtilsTest.java +++ b/linkis-commons/linkis-common/src/test/java/org/apache/linkis/common/utils/ByteTimeUtilsTest.java @@ -17,31 +17,159 @@ package org.apache.linkis.common.utils; +import java.util.function.Function; + +import com.google.common.collect.ImmutableMap; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; -import static org.junit.jupiter.api.Assertions.*; - class ByteTimeUtilsTest { + private static final ImmutableMap> opFunction = + ImmutableMap.>builder() + .put("byteStringAsBytes", tar -> ByteTimeUtils.byteStringAsBytes(tar)) + .put("byteStringAsKb", tar -> ByteTimeUtils.byteStringAsKb(tar)) + .put("byteStringAsMb", tar -> ByteTimeUtils.byteStringAsMb(tar)) + .put("byteStringAsGb", tar -> ByteTimeUtils.byteStringAsGb(tar)) + .build(); + + private static final ImmutableMap convertToByte = + ImmutableMap.builder() + .put("1", 1l) + .put("1b", 1l) + .put("1B", 1l) + .put("1k", 1024l) + .put("1K", 1024l) + .put("1kb", 1024l) + .put("1Kb", 1024l) + .put("1kB", 1024l) + .put("1KB", 1024l) + .put("1m", 1024l * 1024l) + .put("1M", 1024l * 1024l) + .put("1mb", 1024l * 1024l) + .put("1Mb", 1024l * 1024l) + .put("1mB", 1024l * 1024l) + .put("1MB", 1024l * 1024l) + .put("1g", 1024l * 1024l * 1024l) + .put("1G", 1024l * 1024l * 1024l) + .put("1gb", 1024l * 1024l * 1024l) + .put("1gB", 1024l * 1024l * 1024l) + .put("1Gb", 1024l * 1024l * 1024l) + .put("1GB", 1024l * 1024l * 1024l) + .put("1t", 1024l * 1024l * 1024l * 1024l) + .put("1T", 1024l * 1024l * 1024l * 1024l) + .put("1tb", 1024l * 1024l * 1024l * 1024l) + .put("1Tb", 1024l * 1024l * 1024l * 1024l) + .put("1tB", 1024l * 1024l * 1024l * 1024l) + .put("1TB", 1024l * 1024l * 1024l * 1024l) + .put("1p", 1024l * 1024l * 1024l * 1024l * 1024l) + .put("1P", 1024l * 1024l * 1024l * 1024l * 1024l) + .put("1pb", 1024l * 1024l * 1024l * 1024l * 1024l) + .put("1Pb", 1024l * 1024l * 1024l * 1024l * 1024l) + .put("1pB", 1024l * 1024l * 1024l * 1024l * 1024l) + .put("1PB", 1024l * 1024l * 1024l * 1024l * 1024l) + .build(); + + private static final ImmutableMap convertToKB = + ImmutableMap.builder() + .put("1", 1l) + .put("1024b", 1l) + .put("1024B", 1l) + .put("1k", 1l) + .put("1K", 1l) + .put("1kb", 1l) + .put("1Kb", 1l) + .put("1kB", 1l) + .put("1KB", 1l) + .put("1m", 1024l) + .put("1M", 1024l) + .put("1mb", 1024l) + .put("1Mb", 1024l) + .put("1mB", 1024l) + .put("1MB", 1024l) + .put("1g", 1024l * 1024l) + .put("1G", 1024l * 1024l) + .put("1gb", 1024l * 1024l) + .put("1gB", 1024l * 1024l) + .put("1Gb", 1024l * 1024l) + .put("1GB", 1024l * 1024l) + .build(); + + private static final ImmutableMap convertToMB = + ImmutableMap.builder() + .put("1", 1l) + .put("1024k", 1l) + .put("1024K", 1l) + .put("1024kb", 1l) + .put("1024Kb", 1l) + .put("1024kB", 1l) + .put("1024KB", 1l) + .put("1m", 1l) + .put("1M", 1l) + .put("1mb", 1l) + .put("1Mb", 1l) + .put("1mB", 1l) + .put("1MB", 1l) + .put("1g", 1024l) + .put("1G", 1024l) + .put("1gb", 1024l) + .put("1gB", 1024l) + .put("1Gb", 1024l) + .put("1GB", 1024l) + .build(); + + private static final ImmutableMap convertToGB = + ImmutableMap.builder() + .put("1", 1l) + .put("1024m", 1l) + .put("1024M", 1l) + .put("1024mb", 1l) + .put("1024Mb", 1l) + .put("1024mB", 1l) + .put("1024MB", 1l) + .put("1g", 1l) + .put("1G", 1l) + .put("1gb", 1l) + .put("1gB", 1l) + .put("1Gb", 1l) + .put("1GB", 1l) + .put("1t", 1024l) + .put("1T", 1024l) + .put("1tb", 1024l) + .put("1Tb", 1024l) + .put("1tB", 1024l) + .put("1TB", 1024l) + .build(); + @Test - void byteStringAsBytes() {} + void byteStringAsBytes() { + convertToByte.forEach( + (k, v) -> Assertions.assertEquals(opFunction.get("byteStringAsBytes").apply(k), v)); + Assertions.assertThrows( + IllegalArgumentException.class, () -> opFunction.get("byteStringAsBytes").apply("1A")); + } @Test - void byteStringAsKb() {} + void byteStringAsKb() { + convertToKB.forEach( + (k, v) -> Assertions.assertEquals(opFunction.get("byteStringAsKb").apply(k), v)); + Assertions.assertThrows( + IllegalArgumentException.class, () -> opFunction.get("byteStringAsKb").apply("1a")); + } @Test - void byteStringAsMb() {} + void byteStringAsMb() { + convertToMB.forEach( + (k, v) -> Assertions.assertEquals(opFunction.get("byteStringAsMb").apply(k), v)); + Assertions.assertThrows( + IllegalArgumentException.class, () -> opFunction.get("byteStringAsMb").apply("1c")); + } @Test void byteStringAsGb() { - Long res = ByteTimeUtils.byteStringAsGb("1G"); - Assertions.assertEquals(res, 1); - + convertToGB.forEach( + (k, v) -> Assertions.assertEquals(opFunction.get("byteStringAsGb").apply(k), v)); Assertions.assertThrows( - NullPointerException.class, - () -> { - ByteTimeUtils.byteStringAsGb("512"); - }); + IllegalArgumentException.class, () -> opFunction.get("byteStringAsGb").apply("1C")); } } From 658c85e7e4fe8e88dcf92c368d2b558a88be6399 Mon Sep 17 00:00:00 2001 From: peacewong Date: Fri, 28 Apr 2023 13:00:33 +0800 Subject: [PATCH 087/261] add log --- .../jobhistory/service/impl/JobHistoryQueryServiceImpl.scala | 2 ++ 1 file changed, 2 insertions(+) diff --git a/linkis-public-enhancements/linkis-jobhistory/src/main/scala/org/apache/linkis/jobhistory/service/impl/JobHistoryQueryServiceImpl.scala b/linkis-public-enhancements/linkis-jobhistory/src/main/scala/org/apache/linkis/jobhistory/service/impl/JobHistoryQueryServiceImpl.scala index e0faea535c..d906c60a8c 100644 --- a/linkis-public-enhancements/linkis-jobhistory/src/main/scala/org/apache/linkis/jobhistory/service/impl/JobHistoryQueryServiceImpl.scala +++ b/linkis-public-enhancements/linkis-jobhistory/src/main/scala/org/apache/linkis/jobhistory/service/impl/JobHistoryQueryServiceImpl.scala @@ -359,6 +359,7 @@ class JobHistoryQueryServiceImpl extends JobHistoryQueryService with Logging { if (StringUtils.isBlank(cacheKey)) { getCountUndoneTasks(username, creator, sDate, eDate, engineType, startJobId) } else { + logger.info("From cache to get un Done task {}", cacheKey) unDoneTaskCache.get( cacheKey, new Callable[Integer] { @@ -378,6 +379,7 @@ class JobHistoryQueryServiceImpl extends JobHistoryQueryService with Logging { engineType: String, startJobId: lang.Long ): Integer = { + logger.info("Get count undone Tasks {}, {}, {}", username, creator, engineType) val statusList: util.List[String] = new util.ArrayList[String]() statusList.add(TaskStatus.Running.toString) statusList.add(TaskStatus.Inited.toString) From f3476abbf795c31f1e8a99e0386b5ccdf309cb6a Mon Sep 17 00:00:00 2001 From: HmhWz <934178858@qq.com> Date: Fri, 28 Apr 2023 16:52:16 +0800 Subject: [PATCH 088/261] Parse to specific instance for flow/entrance/kill in DSSGatewayParser. --- .../org/apache/linkis/gateway/dss/parser/DSSGatewayParser.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/linkis-spring-cloud-services/linkis-service-gateway/linkis-gateway-server-support/src/main/scala/org/apache/linkis/gateway/dss/parser/DSSGatewayParser.scala b/linkis-spring-cloud-services/linkis-service-gateway/linkis-gateway-server-support/src/main/scala/org/apache/linkis/gateway/dss/parser/DSSGatewayParser.scala index 0423304129..10e8fceda0 100644 --- a/linkis-spring-cloud-services/linkis-service-gateway/linkis-gateway-server-support/src/main/scala/org/apache/linkis/gateway/dss/parser/DSSGatewayParser.scala +++ b/linkis-spring-cloud-services/linkis-service-gateway/linkis-gateway-server-support/src/main/scala/org/apache/linkis/gateway/dss/parser/DSSGatewayParser.scala @@ -284,6 +284,6 @@ object DSSGatewayParser { val APPCONN_URL_DEFAULT_REGEX = (APPCONN_HEADER + "([^/]+).+").r val DSS_URL_FLOW_QUERY_PREFIX = - (DSS_HEADER + "flow/entrance/" + "([^/]+)/" + "(status|execution)").r + (DSS_HEADER + "flow/entrance/" + "([^/]+)/" + "(status|execution|kill)").r } From 0b8a8fdbff08db46f966f82cbbf5acac5168c2e0 Mon Sep 17 00:00:00 2001 From: casionone Date: Thu, 4 May 2023 00:08:00 +0800 Subject: [PATCH 089/261] version num to 1.1.11-wedatasphere --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 0eaa57f1f5..fa691bc40c 100644 --- a/pom.xml +++ b/pom.xml @@ -103,7 +103,7 @@ - 1.1.10-wedatasphere + 1.1.11-wedatasphere 2.9.2 2.4.3 2.7.2 From 474c3fff65ed88b02ccf2c1f1ee4a9e94337c122 Mon Sep 17 00:00:00 2001 From: Casion Date: Thu, 4 May 2023 00:14:25 +0800 Subject: [PATCH 090/261] fix:select exclude route label is empty (#147) --- .../main/resources/mapper/common/InsLabelRelationMapper.xml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/linkis-public-enhancements/linkis-instance-label/linkis-instance-label-server/src/main/resources/mapper/common/InsLabelRelationMapper.xml b/linkis-public-enhancements/linkis-instance-label/linkis-instance-label-server/src/main/resources/mapper/common/InsLabelRelationMapper.xml index 2a5d8a3d45..f11c80624d 100644 --- a/linkis-public-enhancements/linkis-instance-label/linkis-instance-label-server/src/main/resources/mapper/common/InsLabelRelationMapper.xml +++ b/linkis-public-enhancements/linkis-instance-label/linkis-instance-label-server/src/main/resources/mapper/common/InsLabelRelationMapper.xml @@ -197,7 +197,9 @@ - \ No newline at end of file From 886daa7bd3c1e13d0407d69a88b7c71e6354db9a Mon Sep 17 00:00:00 2001 From: Casion Date: Fri, 5 May 2023 22:59:18 +0800 Subject: [PATCH 091/261] added askEngineHook (#143) * 1. added askEngineHook * 1. optimize return value for hook * 1. added two lost files --------- Co-authored-by: shangda --- .../manager/am/hook/AskEngineHook.scala | 25 ++++++++++++ .../am/hook/AskEngineHookContext.scala | 26 +++++++++++++ .../service/em/DefaultEMEngineService.scala | 6 ++- .../DefaultEngineAskEngineService.scala | 38 ++++++++++++++++++- 4 files changed, 93 insertions(+), 2 deletions(-) create mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/hook/AskEngineHook.scala create mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/hook/AskEngineHookContext.scala diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/hook/AskEngineHook.scala b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/hook/AskEngineHook.scala new file mode 100644 index 0000000000..03bb7126aa --- /dev/null +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/hook/AskEngineHook.scala @@ -0,0 +1,25 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.manager.am.hook + +trait AskEngineHook { + + @throws(classOf[Exception]) + def doHook(ctx: AskEngineHookContext): Unit + +} diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/hook/AskEngineHookContext.scala b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/hook/AskEngineHookContext.scala new file mode 100644 index 0000000000..38aa1789b0 --- /dev/null +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/hook/AskEngineHookContext.scala @@ -0,0 +1,26 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.manager.am.hook + +import org.apache.linkis.manager.common.protocol.engine.EngineAskRequest +import org.apache.linkis.rpc.Sender + +class AskEngineHookContext(request: EngineAskRequest, sender: Sender) { + def getRequest(): EngineAskRequest = request + def getSender(): Sender = sender +} diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/service/em/DefaultEMEngineService.scala b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/service/em/DefaultEMEngineService.scala index 918faf912f..aba78a85ba 100644 --- a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/service/em/DefaultEMEngineService.scala +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/service/em/DefaultEMEngineService.scala @@ -116,7 +116,11 @@ class DefaultEMEngineService extends EMEngineService with Logging { val filterInstanceAndLabel = if (emInstanceLabelOption.isDefined) { val emInstanceLabel = emInstanceLabelOption.get.asInstanceOf[EMInstanceLabel] logger.info(s"use emInstanceLabel , will be route to ${emInstanceLabel.getServiceInstance}") - if (!instanceAndLabels.asScala.exists(_._1.equals(emInstanceLabel.getServiceInstance))) { + if ( + !instanceAndLabels.asScala.exists( + _._1.getServiceInstance.equals(emInstanceLabel.getServiceInstance) + ) + ) { throw new AMErrorException( AMConstant.EM_ERROR_CODE, s"You specified em ${emInstanceLabel.getServiceInstance}, but the corresponding EM does not exist in the Manager" diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/service/engine/DefaultEngineAskEngineService.scala b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/service/engine/DefaultEngineAskEngineService.scala index 71fe772215..94754e9cd8 100644 --- a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/service/engine/DefaultEngineAskEngineService.scala +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/service/engine/DefaultEngineAskEngineService.scala @@ -21,6 +21,7 @@ import org.apache.linkis.common.exception.LinkisRetryException import org.apache.linkis.common.utils.{Logging, Utils} import org.apache.linkis.governance.common.utils.{JobUtils, LoggerUtils} import org.apache.linkis.manager.am.conf.AMConfiguration +import org.apache.linkis.manager.am.hook.{AskEngineHook, AskEngineHookContext} import org.apache.linkis.manager.common.constant.AMConstant import org.apache.linkis.manager.common.protocol.engine._ import org.apache.linkis.manager.label.constant.LabelKeyConstant @@ -29,7 +30,7 @@ import org.apache.linkis.rpc.message.annotation.Receiver import org.apache.commons.lang3.exception.ExceptionUtils -import org.springframework.beans.factory.annotation.Autowired +import org.springframework.beans.factory.annotation.{Autowired, Qualifier} import org.springframework.stereotype.Service import java.net.SocketTimeoutException @@ -55,6 +56,11 @@ class DefaultEngineAskEngineService @Autowired private var engineSwitchService: EngineSwitchService = _ + @Autowired(required = false) + @Qualifier + /* hook的实现类必须加上@Qualifier注解才能生效 */ + var hooks: Array[AskEngineHook] = _ + private val idCreator = new AtomicInteger() private val idPrefix = Sender.getThisServiceInstance.getInstance @@ -67,6 +73,36 @@ class DefaultEngineAskEngineService @Receiver override def askEngine(engineAskRequest: EngineAskRequest, sender: Sender): Any = { + + if (hooks != null && hooks.size > 0) { + val ctx = new AskEngineHookContext(engineAskRequest, sender) + + /** hook中抛异常会阻断 */ + hooks.foreach(h => + Utils.tryCatch(h.doHook(ctx)) { t => + { + val engineAskAsyncId = getAsyncId + val retryFlag = t match { + case _: LinkisRetryException => true + case _: RetryableException => true + case _ => + ExceptionUtils.getRootCause(t) match { + case _: SocketTimeoutException => true + case _: TimeoutException => true + case _ => + false + } + } + return EngineCreateError( + engineAskAsyncId, + ExceptionUtils.getRootCauseMessage(t), + retryFlag + ) + } + } + ) + } + val taskId = JobUtils.getJobIdFromStringMap(engineAskRequest.getProperties) LoggerUtils.setJobIdMDC(taskId) logger.info(s"received task: $taskId, engineAskRequest $engineAskRequest") From 76b20b55951d00939db0bc45b4d206be93b54f0d Mon Sep 17 00:00:00 2001 From: Casion Date: Fri, 5 May 2023 22:59:30 +0800 Subject: [PATCH 092/261] 1.1.11 webank bdpclient (#142) * 1. refactored linkis-cli * 1. added errorCode and errorDesc * 1. added errorCode and errorDesc * 1. added errorCode and errorDesc * 1. modified creators * 1. fixed bugs --------- Co-authored-by: shangda --- .../linkis-cli/linkis-cli-application/pom.xml | 87 --- .../application/LinkisClientApplication.java | 341 ------------ .../cli/application/data/FinishedData.java | 47 -- .../cli/application/data/ProcessedData.java | 52 -- .../application/interactor/job/LinkisJob.java | 26 - .../interactor/job/LinkisManageJob.java | 522 ------------------ .../interactor/job/LinkisOnceJob.java | 308 ----------- .../interactor/job/LinkisSubmitJob.java | 282 ---------- .../job/builder/LinkisManageJobBuilder.java | 140 ----- .../job/builder/LinkisOnceJobBuilder.java | 239 -------- .../job/builder/LinkisSubmitJobBuilder.java | 241 -------- .../interactor/job/data/LinkisJobData.java | 40 -- .../job/data/LinkisJobDataImpl.java | 514 ----------------- .../interactor/job/data/LinkisLogData.java | 51 -- .../job/data/LinkisOnceJobData.java | 242 -------- .../interactor/job/data/LinkisResultData.java | 39 -- .../job/data/SimpleOnceJobAdapter.java | 377 ------------- .../job/subtype/LinkisSubmitSubType.java | 35 -- .../listener/TriggerEventListener.java | 35 -- .../application/operator/OperatorUtils.java | 70 --- .../present/LinkisResultInfoPresenter.java | 114 ---- .../present/LinkisResultPresenter.java | 232 -------- .../present/model/LinkisJobInfoModel.java | 97 ---- .../present/model/LinkisJobKillModel.java | 63 --- .../present/model/LinkisResultInfoModel.java | 84 --- .../model/LinkisSubmitResultModel.java | 56 -- .../cli/application/suite/ExecutionSuite.java | 61 -- .../suite/ExecutionSuiteFactory.java | 225 -------- .../linkis/cli/application/utils/Utils.java | 52 -- .../command/template/ProcessInputUtil.java | 127 ----- .../execution/LinkisExecutorTest.java | 113 ---- .../interactor/job/LinkisJobBuilderTest.java | 106 ---- .../job/subtype/LinkisSubmitSubTypeTest.java | 33 -- .../linkis-cli/linkis-cli-common/pom.xml | 29 - .../common/entity/execution/Execution.java | 35 -- .../linkis/cli/common/entity/job/Job.java | 55 -- .../linkis/cli/common/entity/job/JobData.java | 34 -- .../cli/common/entity/job/JobDescription.java | 21 - .../common/entity/operator/JobOperator.java | 20 - .../cli/common/entity/present/PresentWay.java | 20 - .../common/entity/result/ExecutionResult.java | 37 -- .../common/entity/result/ExecutionStatus.java | 20 - .../common/entity/result/ResultHandler.java | 22 - .../exception/handler/ExceptionHandler.java | 22 - .../exception/error/ErrorLevelTest.java | 42 -- .../linkis-cli/linkis-cli-core/pom.xml | 43 -- .../cli/core/builder/AbstractBuilder.java | 54 -- .../core/builder/BuildableByVarAccess.java | 56 -- .../linkis/cli/core/builder/Builder.java | 22 - .../cli/core/constants/CommonConstants.java | 38 -- .../linkis/cli/core/constants/CommonKeys.java | 20 - .../handler/CommandExceptionHandler.java | 46 -- .../handler/DefaultExceptionHandler.java | 83 --- .../interactor/execution/AsyncSubmission.java | 109 ---- .../cli/core/interactor/execution/Help.java | 59 -- .../interactor/execution/JobManagement.java | 96 ---- .../interactor/execution/SyncSubmission.java | 195 ------- .../cli/core/interactor/job/AbstractJob.java | 77 --- .../core/interactor/job/AsyncBackendJob.java | 38 -- .../cli/core/interactor/job/JobBuilder.java | 52 -- .../core/interactor/job/LogAccessibleJob.java | 30 - .../interactor/job/ManagableBackendJob.java | 42 -- .../interactor/job/ResultAccessibleJob.java | 29 - .../core/interactor/job/SyncBackendJob.java | 29 - .../core/interactor/job/TerminatableJob.java | 29 - .../core/interactor/job/TerminateToken.java | 32 -- .../properties/PropertiesLoader.java | 35 -- .../properties/reader/SysEnvReader.java | 64 --- .../properties/reader/SysPropsReader.java | 62 --- .../result/DefaultResultHandler.java | 39 -- .../result/ExecutionResultImpl.java | 68 --- .../result/ExecutionStatusEnum.java | 34 -- .../result/PresentResultHandler.java | 78 --- .../interactor/validate/PropsValidator.java | 31 -- .../cli/core/interactor/var/SysVarAccess.java | 114 ---- .../cli/core/operator/JobOperatorFactory.java | 93 ---- .../core/present/DefaultStdOutPresenter.java | 35 -- .../linkis/cli/core/present/PresentMode.java | 22 - .../cli/core/present/PresentWayImpl.java | 51 -- .../present/display/DisplayOperFactory.java | 50 -- .../core/present/display/StdOutWriter.java | 43 -- .../present/display/data/DisplayData.java | 20 - .../present/display/data/FileDisplayData.java | 47 -- .../display/data/StdoutDisplayData.java | 30 - .../linkis/cli/core/utils/CommonUtils.java | 82 --- .../core/constants/CommonConstantsTest.java | 60 -- .../cli/core/constants/TestConstants.java | 100 ---- .../exception/error/CommonErrMsgTest.java | 41 -- .../handler/CommandExceptionHandlerTest.java | 75 --- .../core/interactor/command/TestCmdType.java | 58 -- .../command/fitter/FitterUtilsTest.java | 34 -- .../command/fitter/SingleTplFitterTest.java | 119 ---- .../command/parser/SingleCmdParserTest.java | 177 ------ .../command/template/TestParamMapper.java | 35 -- .../template/TestSparkCmdTemplate.java | 268 --------- .../properties/StdPropsLoaderTest.java | 119 ---- .../core/interactor/var/StdVarAccessTest.java | 298 ---------- .../cli/core/present/PresentModeImplTest.java | 36 -- .../linkis/cli/core/utils/TestUtils.java | 73 --- .../test/resources/conf/linkis-cli.properties | 34 -- .../src/test/resources/conf/user.properties | 19 - .../src/test/resources/log4j2.xml | 50 -- .../linkis-client/linkis-cli/pom.xml | 61 +- .../src/main/assembly/distribution.xml | 2 +- .../linkis/cli/application/CtxBuilder.java | 185 +++++++ .../application/LinkisClientApplication.java | 189 +++++++ .../application/constants/CliConstants.java} | 32 +- .../cli/application/constants/CliKeys.java} | 8 +- .../constants/LinkisConstants.java | 0 .../cli/application/constants/LinkisKeys.java | 1 + .../entity/command/CmdOption.java | 2 +- .../entity/command/CmdTemplate.java | 2 +- .../application}/entity/command/CmdType.java | 2 +- .../entity/command/ParamItem.java | 2 +- .../application}/entity/command/Params.java | 8 +- .../application/entity/context/CliCtx.java} | 25 +- .../cli/application/entity/job/Job.java} | 12 +- .../application/entity/job/JobResult.java} | 11 +- .../application}/entity/job/JobStatus.java | 2 +- .../application/entity/operator/JobOper.java} | 4 +- .../application}/entity/present/Model.java | 2 +- .../entity/present/Presenter.java | 4 +- .../application}/entity/var/VarAccess.java | 4 +- .../exception/BuilderException.java | 7 +- .../exception/CommandException.java | 9 +- .../exception/LinkisClientException.java | 2 +- .../LinkisClientExecutionException.java | 7 +- .../LinkisClientRuntimeException.java | 6 +- .../exception/PresenterException.java | 7 +- .../exception/PropsException.java | 7 +- .../exception/TransformerException.java | 7 +- .../exception/UnknownException.java | 7 +- .../exception/ValidateException.java | 7 +- .../exception/VarAccessException.java | 7 +- .../exception/error/CommonErrMsg.java | 4 +- .../exception/error/ErrorLevel.java | 2 +- .../exception/error/ErrorMsg.java | 2 +- .../interactor/command/CliCmdType.java} | 12 +- .../command/CmdTemplateFactory.java | 12 +- .../interactor/command/SpecialMap.java | 2 +- .../command/fitter/AbstractFitter.java | 26 +- .../interactor/command/fitter/Fitter.java | 6 +- .../command/fitter/FitterResult.java | 4 +- .../command/fitter/FitterUtils.java | 2 +- .../command/fitter/SingleTplFitter.java | 14 +- .../command/parser/AbstarctParser.java | 28 +- .../interactor/command/parser/Parser.java | 6 +- .../command/parser/SingleCmdParser.java | 28 +- .../command/parser/result/ParseResult.java | 6 +- .../parser/transformer/ParamKeyMapper.java | 14 +- .../command/template/AbstractCmdTemplate.java | 32 +- .../template/UniversalCmdTemplate.java | 151 ++--- .../converter/AbstractStringConverter.java | 2 +- .../template}/converter/Converter.java | 2 +- .../converter/PredefinedStringConverters.java | 14 +- .../command/template/option/BaseOption.java | 6 +- .../command/template/option/Flag.java | 4 +- .../command/template/option/MapOption.java | 8 +- .../command/template/option/Parameter.java | 4 +- .../template/option/SpecialMapOption.java | 4 +- .../command/template/option/StdOption.java | 4 +- .../interactor/context/CliCtxImpl.java | 60 ++ .../interactor/job/common/JobKiller.java | 121 ++++ .../interactor/job/common/KeyParser.java} | 20 +- .../interactor/job/common/KillResult.java | 61 ++ .../job/common}/LinkisJobStatus.java | 4 +- .../interactor/job/common/LogData.java | 112 ++++ .../interactor/job/common/LogRetriever.java | 202 +++++++ .../interactor/job/common/ResultData.java | 165 ++++++ .../job/common/ResultRetriever.java | 186 +++++++ .../interactor/job/common/ResultSet.java} | 10 +- .../interactor/job/help/HelpJob.java | 62 +++ .../job/interactive/InteractiveJob.java | 260 +++++++++ .../job/interactive/InteractiveJobDesc.java} | 6 +- .../InteractiveJobDescBuilder.java | 177 ++++++ .../job/interactive/InteractiveJobResult.java | 61 ++ .../interactor/job/jobcmd/JobCmdDesc.java} | 15 +- .../job/jobcmd/JobCmdDescBuilder.java | 67 +++ .../interactor/job/jobcmd/JobCmdJob.java | 212 +++++++ .../job/jobcmd/JobCmdJobResult.java | 61 ++ .../interactor/job/jobcmd/JobCmdSubType.java} | 9 +- .../interactor/job/once/LinkisOnceJob.java | 106 ++++ .../interactor/job/once/OnceJobResult.java | 61 ++ .../interactor/job/version/VersionJob.java | 48 ++ .../job/version/VersionJobResult.java} | 33 +- .../properties/ClientProperties.java | 2 +- .../properties/PropertiesLoader.java} | 21 +- .../properties/PropsFilesScanner.java | 16 +- .../properties/reader/PropertiesReader.java | 2 +- .../properties/reader/PropsFileReader.java | 8 +- .../validate/InteractiveDescValidator.java} | 69 +-- .../validate/JobCmdDescValidator.java} | 26 +- .../validate/OnceDescValidator.java} | 72 +-- .../interactor/validate/ParamValidator.java | 10 +- .../validate/ParsedTplValidator.java | 29 +- .../validate/UJESContextValidator.java | 22 +- .../interactor/var/VarAccessImpl.java} | 37 +- .../observer/event/FetchResultEvent.java} | 2 +- .../observer/event/LinkisClientEvent.java | 0 .../observer/event/LogStartEvent.java | 0 .../observer/event/SingleObserverEvent.java | 0 .../listener/LinkisClientListener.java | 0 .../application/operator/JobOperBuilder.java} | 9 +- .../application/operator/OperManager.java} | 30 +- .../operator/once/LinkisNodeStatus.java | 0 .../operator/once/OnceJobConstants.java | 0 .../operator/once/OnceJobDesc.java} | 28 +- .../operator/once/OnceJobOper.java | 166 ++++++ .../operator/once/OnceOperBuilder.java | 169 ++++++ .../operator/ujes/LinkisJobOper.java} | 237 +++----- .../operator/ujes/LinkisOperBuilder.java} | 29 +- .../ujes/LinkisOperResultAdapter.java | 6 +- .../operator/ujes/UJESClientContext.java | 0 .../operator/ujes/UJESClientFactory.java | 58 +- .../operator/ujes/UJESConstants.java | 0 .../operator/ujes/UJESResultAdapter.java | 76 ++- .../operator/ujes/result/OpenLogResult2.java | 0 .../ujes/result/ResultSetResult2.java | 0 .../operator/ujes/result/UJESResult.java | 0 .../application/present/HelpPresenter.java} | 43 +- .../present/JobInfoPresenter.java} | 22 +- .../application/present/LogPresenter.java} | 45 +- .../application/present/ResultPresenter.java | 301 ++++++++++ .../present/file/ResultFileWriter.java} | 27 +- .../present/model/HelpInfoModel.java | 12 +- .../present/model/LinkisJobInfoModel.java | 89 +++ .../present/model/LinkisLogModel.java | 18 +- .../present/model/LinkisResultModel.java | 40 +- .../cli/application/utils/CliUtils.java} | 117 +++- .../cli/application/utils/LoggerManager.java} | 4 +- .../application/utils/SchedulerManager.java} | 8 +- .../src/main/resources/version.properties | 1 + .../LinkisClientApplicationTest.java | 40 +- .../constants/CliConstantsTest.java} | 28 +- .../application/constants/CliKeysTest.java} | 32 +- .../constants/LinkisConstantsTest.java | 0 .../application/constants/LinkisKeysTest.java | 0 .../application/constants/TestConstants.java | 0 .../interactor/command/CliCmdTypeTest.java} | 8 +- .../command/template/TestCmdType.java | 2 +- .../command/template/TestParamMapper.java | 18 +- .../template/TestSparkCmdTemplate.java | 13 +- .../interactor/job/LinkisJobStatusTest.java | 2 + .../job/builder/KeyParserTest.java} | 9 +- .../job/subtype/JobCmdSubTypeTest.java} | 16 +- .../operator/once/OnceJobConstantsTest.java | 0 .../operator/ujes/UJESConstantsTest.java | 0 .../cli/application/utils/UtilsTest.java | 2 +- .../test/resources/conf/linkis-cli.properties | 0 .../src/test/resources/conf/user.properties | 0 .../src/test/resources/linkis-cli.properties | 0 .../src/test/resources/log4j2.xml | 0 .../src/test/resources/testScala.scala | 0 pom.xml | 6 +- 254 files changed, 4194 insertions(+), 10163 deletions(-) delete mode 100644 linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/pom.xml delete mode 100644 linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/LinkisClientApplication.java delete mode 100644 linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/data/FinishedData.java delete mode 100644 linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/data/ProcessedData.java delete mode 100644 linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/interactor/job/LinkisJob.java delete mode 100644 linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/interactor/job/LinkisManageJob.java delete mode 100644 linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/interactor/job/LinkisOnceJob.java delete mode 100644 linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/interactor/job/LinkisSubmitJob.java delete mode 100644 linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/interactor/job/builder/LinkisManageJobBuilder.java delete mode 100644 linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/interactor/job/builder/LinkisOnceJobBuilder.java delete mode 100644 linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/interactor/job/builder/LinkisSubmitJobBuilder.java delete mode 100644 linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/interactor/job/data/LinkisJobData.java delete mode 100644 linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/interactor/job/data/LinkisJobDataImpl.java delete mode 100644 linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/interactor/job/data/LinkisLogData.java delete mode 100644 linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/interactor/job/data/LinkisOnceJobData.java delete mode 100644 linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/interactor/job/data/LinkisResultData.java delete mode 100644 linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/interactor/job/data/SimpleOnceJobAdapter.java delete mode 100644 linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/interactor/job/subtype/LinkisSubmitSubType.java delete mode 100644 linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/observer/listener/TriggerEventListener.java delete mode 100644 linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/operator/OperatorUtils.java delete mode 100644 linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/present/LinkisResultInfoPresenter.java delete mode 100644 linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/present/LinkisResultPresenter.java delete mode 100644 linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/present/model/LinkisJobInfoModel.java delete mode 100644 linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/present/model/LinkisJobKillModel.java delete mode 100644 linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/present/model/LinkisResultInfoModel.java delete mode 100644 linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/present/model/LinkisSubmitResultModel.java delete mode 100644 linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/suite/ExecutionSuite.java delete mode 100644 linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/suite/ExecutionSuiteFactory.java delete mode 100644 linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/utils/Utils.java delete mode 100644 linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/test/java/org/apache/linkis/cli/application/interactor/command/template/ProcessInputUtil.java delete mode 100644 linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/test/java/org/apache/linkis/cli/application/interactor/execution/LinkisExecutorTest.java delete mode 100644 linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/test/java/org/apache/linkis/cli/application/interactor/job/LinkisJobBuilderTest.java delete mode 100644 linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/test/java/org/apache/linkis/cli/application/interactor/job/subtype/LinkisSubmitSubTypeTest.java delete mode 100644 linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-common/pom.xml delete mode 100644 linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-common/src/main/java/org/apache/linkis/cli/common/entity/execution/Execution.java delete mode 100644 linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-common/src/main/java/org/apache/linkis/cli/common/entity/job/Job.java delete mode 100644 linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-common/src/main/java/org/apache/linkis/cli/common/entity/job/JobData.java delete mode 100644 linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-common/src/main/java/org/apache/linkis/cli/common/entity/job/JobDescription.java delete mode 100644 linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-common/src/main/java/org/apache/linkis/cli/common/entity/operator/JobOperator.java delete mode 100644 linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-common/src/main/java/org/apache/linkis/cli/common/entity/present/PresentWay.java delete mode 100644 linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-common/src/main/java/org/apache/linkis/cli/common/entity/result/ExecutionResult.java delete mode 100644 linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-common/src/main/java/org/apache/linkis/cli/common/entity/result/ExecutionStatus.java delete mode 100644 linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-common/src/main/java/org/apache/linkis/cli/common/entity/result/ResultHandler.java delete mode 100644 linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-common/src/main/java/org/apache/linkis/cli/common/exception/handler/ExceptionHandler.java delete mode 100644 linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-common/src/test/java/org/apache/linkis/cli/common/exception/error/ErrorLevelTest.java delete mode 100644 linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/pom.xml delete mode 100644 linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/builder/AbstractBuilder.java delete mode 100644 linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/builder/BuildableByVarAccess.java delete mode 100644 linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/builder/Builder.java delete mode 100644 linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/constants/CommonConstants.java delete mode 100644 linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/constants/CommonKeys.java delete mode 100644 linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/exception/handler/CommandExceptionHandler.java delete mode 100644 linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/exception/handler/DefaultExceptionHandler.java delete mode 100644 linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/execution/AsyncSubmission.java delete mode 100644 linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/execution/Help.java delete mode 100644 linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/execution/JobManagement.java delete mode 100644 linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/execution/SyncSubmission.java delete mode 100644 linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/job/AbstractJob.java delete mode 100644 linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/job/AsyncBackendJob.java delete mode 100644 linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/job/JobBuilder.java delete mode 100644 linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/job/LogAccessibleJob.java delete mode 100644 linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/job/ManagableBackendJob.java delete mode 100644 linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/job/ResultAccessibleJob.java delete mode 100644 linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/job/SyncBackendJob.java delete mode 100644 linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/job/TerminatableJob.java delete mode 100644 linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/job/TerminateToken.java delete mode 100644 linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/properties/PropertiesLoader.java delete mode 100644 linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/properties/reader/SysEnvReader.java delete mode 100644 linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/properties/reader/SysPropsReader.java delete mode 100644 linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/result/DefaultResultHandler.java delete mode 100644 linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/result/ExecutionResultImpl.java delete mode 100644 linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/result/ExecutionStatusEnum.java delete mode 100644 linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/result/PresentResultHandler.java delete mode 100644 linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/validate/PropsValidator.java delete mode 100644 linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/var/SysVarAccess.java delete mode 100644 linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/operator/JobOperatorFactory.java delete mode 100644 linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/present/DefaultStdOutPresenter.java delete mode 100644 linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/present/PresentMode.java delete mode 100644 linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/present/PresentWayImpl.java delete mode 100644 linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/present/display/DisplayOperFactory.java delete mode 100644 linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/present/display/StdOutWriter.java delete mode 100644 linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/present/display/data/DisplayData.java delete mode 100644 linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/present/display/data/FileDisplayData.java delete mode 100644 linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/present/display/data/StdoutDisplayData.java delete mode 100644 linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/utils/CommonUtils.java delete mode 100644 linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/test/java/org/apache/linkis/cli/core/constants/CommonConstantsTest.java delete mode 100644 linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/test/java/org/apache/linkis/cli/core/constants/TestConstants.java delete mode 100644 linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/test/java/org/apache/linkis/cli/core/exception/error/CommonErrMsgTest.java delete mode 100644 linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/test/java/org/apache/linkis/cli/core/exception/handler/CommandExceptionHandlerTest.java delete mode 100644 linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/test/java/org/apache/linkis/cli/core/interactor/command/TestCmdType.java delete mode 100644 linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/test/java/org/apache/linkis/cli/core/interactor/command/fitter/FitterUtilsTest.java delete mode 100644 linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/test/java/org/apache/linkis/cli/core/interactor/command/fitter/SingleTplFitterTest.java delete mode 100644 linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/test/java/org/apache/linkis/cli/core/interactor/command/parser/SingleCmdParserTest.java delete mode 100644 linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/test/java/org/apache/linkis/cli/core/interactor/command/template/TestParamMapper.java delete mode 100644 linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/test/java/org/apache/linkis/cli/core/interactor/command/template/TestSparkCmdTemplate.java delete mode 100644 linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/test/java/org/apache/linkis/cli/core/interactor/properties/StdPropsLoaderTest.java delete mode 100644 linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/test/java/org/apache/linkis/cli/core/interactor/var/StdVarAccessTest.java delete mode 100644 linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/test/java/org/apache/linkis/cli/core/present/PresentModeImplTest.java delete mode 100644 linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/test/java/org/apache/linkis/cli/core/utils/TestUtils.java delete mode 100644 linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/test/resources/conf/linkis-cli.properties delete mode 100644 linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/test/resources/conf/user.properties delete mode 100644 linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/test/resources/log4j2.xml rename linkis-computation-governance/linkis-client/linkis-cli/{linkis-cli-application => }/src/main/assembly/distribution.xml (98%) create mode 100644 linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/CtxBuilder.java create mode 100644 linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/LinkisClientApplication.java rename linkis-computation-governance/linkis-client/linkis-cli/{linkis-cli-application/src/main/java/org/apache/linkis/cli/application/constants/AppConstants.java => src/main/java/org/apache/linkis/cli/application/constants/CliConstants.java} (61%) rename linkis-computation-governance/linkis-client/linkis-cli/{linkis-cli-application/src/main/java/org/apache/linkis/cli/application/constants/AppKeys.java => src/main/java/org/apache/linkis/cli/application/constants/CliKeys.java} (97%) rename linkis-computation-governance/linkis-client/linkis-cli/{linkis-cli-application => }/src/main/java/org/apache/linkis/cli/application/constants/LinkisConstants.java (100%) rename linkis-computation-governance/linkis-client/linkis-cli/{linkis-cli-application => }/src/main/java/org/apache/linkis/cli/application/constants/LinkisKeys.java (97%) rename linkis-computation-governance/linkis-client/linkis-cli/{linkis-cli-common/src/main/java/org/apache/linkis/cli/common => src/main/java/org/apache/linkis/cli/application}/entity/command/CmdOption.java (95%) rename linkis-computation-governance/linkis-client/linkis-cli/{linkis-cli-common/src/main/java/org/apache/linkis/cli/common => src/main/java/org/apache/linkis/cli/application}/entity/command/CmdTemplate.java (94%) rename linkis-computation-governance/linkis-client/linkis-cli/{linkis-cli-common/src/main/java/org/apache/linkis/cli/common => src/main/java/org/apache/linkis/cli/application}/entity/command/CmdType.java (93%) rename linkis-computation-governance/linkis-client/linkis-cli/{linkis-cli-common/src/main/java/org/apache/linkis/cli/common => src/main/java/org/apache/linkis/cli/application}/entity/command/ParamItem.java (96%) rename linkis-computation-governance/linkis-client/linkis-cli/{linkis-cli-common/src/main/java/org/apache/linkis/cli/common => src/main/java/org/apache/linkis/cli/application}/entity/command/Params.java (91%) rename linkis-computation-governance/linkis-client/linkis-cli/{linkis-cli-application/src/main/java/org/apache/linkis/cli/application/data/PreparedData.java => src/main/java/org/apache/linkis/cli/application/entity/context/CliCtx.java} (62%) rename linkis-computation-governance/linkis-client/linkis-cli/{linkis-cli-core/src/main/java/org/apache/linkis/cli/core/present/display/DisplayOperator.java => src/main/java/org/apache/linkis/cli/application/entity/job/Job.java} (79%) rename linkis-computation-governance/linkis-client/linkis-cli/{linkis-cli-common/src/main/java/org/apache/linkis/cli/common/entity/job/JobSubType.java => src/main/java/org/apache/linkis/cli/application/entity/job/JobResult.java} (80%) rename linkis-computation-governance/linkis-client/linkis-cli/{linkis-cli-common/src/main/java/org/apache/linkis/cli/common => src/main/java/org/apache/linkis/cli/application}/entity/job/JobStatus.java (95%) rename linkis-computation-governance/linkis-client/linkis-cli/{linkis-cli-common/src/main/java/org/apache/linkis/cli/common/entity/version/Version.java => src/main/java/org/apache/linkis/cli/application/entity/operator/JobOper.java} (90%) rename linkis-computation-governance/linkis-client/linkis-cli/{linkis-cli-common/src/main/java/org/apache/linkis/cli/common => src/main/java/org/apache/linkis/cli/application}/entity/present/Model.java (93%) rename linkis-computation-governance/linkis-client/linkis-cli/{linkis-cli-common/src/main/java/org/apache/linkis/cli/common => src/main/java/org/apache/linkis/cli/application}/entity/present/Presenter.java (88%) rename linkis-computation-governance/linkis-client/linkis-cli/{linkis-cli-common/src/main/java/org/apache/linkis/cli/common => src/main/java/org/apache/linkis/cli/application}/entity/var/VarAccess.java (94%) rename linkis-computation-governance/linkis-client/linkis-cli/{linkis-cli-core/src/main/java/org/apache/linkis/cli/core => src/main/java/org/apache/linkis/cli/application}/exception/BuilderException.java (83%) rename linkis-computation-governance/linkis-client/linkis-cli/{linkis-cli-core/src/main/java/org/apache/linkis/cli/core => src/main/java/org/apache/linkis/cli/application}/exception/CommandException.java (85%) rename linkis-computation-governance/linkis-client/linkis-cli/{linkis-cli-common/src/main/java/org/apache/linkis/cli/common => src/main/java/org/apache/linkis/cli/application}/exception/LinkisClientException.java (94%) rename linkis-computation-governance/linkis-client/linkis-cli/{linkis-cli-core/src/main/java/org/apache/linkis/cli/core => src/main/java/org/apache/linkis/cli/application}/exception/LinkisClientExecutionException.java (84%) rename linkis-computation-governance/linkis-client/linkis-cli/{linkis-cli-common/src/main/java/org/apache/linkis/cli/common => src/main/java/org/apache/linkis/cli/application}/exception/LinkisClientRuntimeException.java (94%) rename linkis-computation-governance/linkis-client/linkis-cli/{linkis-cli-core/src/main/java/org/apache/linkis/cli/core => src/main/java/org/apache/linkis/cli/application}/exception/PresenterException.java (83%) rename linkis-computation-governance/linkis-client/linkis-cli/{linkis-cli-core/src/main/java/org/apache/linkis/cli/core => src/main/java/org/apache/linkis/cli/application}/exception/PropsException.java (83%) rename linkis-computation-governance/linkis-client/linkis-cli/{linkis-cli-core/src/main/java/org/apache/linkis/cli/core => src/main/java/org/apache/linkis/cli/application}/exception/TransformerException.java (83%) rename linkis-computation-governance/linkis-client/linkis-cli/{linkis-cli-core/src/main/java/org/apache/linkis/cli/core => src/main/java/org/apache/linkis/cli/application}/exception/UnknownException.java (83%) rename linkis-computation-governance/linkis-client/linkis-cli/{linkis-cli-core/src/main/java/org/apache/linkis/cli/core => src/main/java/org/apache/linkis/cli/application}/exception/ValidateException.java (83%) rename linkis-computation-governance/linkis-client/linkis-cli/{linkis-cli-core/src/main/java/org/apache/linkis/cli/core => src/main/java/org/apache/linkis/cli/application}/exception/VarAccessException.java (83%) rename linkis-computation-governance/linkis-client/linkis-cli/{linkis-cli-core/src/main/java/org/apache/linkis/cli/core => src/main/java/org/apache/linkis/cli/application}/exception/error/CommonErrMsg.java (96%) rename linkis-computation-governance/linkis-client/linkis-cli/{linkis-cli-common/src/main/java/org/apache/linkis/cli/common => src/main/java/org/apache/linkis/cli/application}/exception/error/ErrorLevel.java (96%) rename linkis-computation-governance/linkis-client/linkis-cli/{linkis-cli-common/src/main/java/org/apache/linkis/cli/common => src/main/java/org/apache/linkis/cli/application}/exception/error/ErrorMsg.java (93%) rename linkis-computation-governance/linkis-client/linkis-cli/{linkis-cli-application/src/main/java/org/apache/linkis/cli/application/interactor/command/LinkisCmdType.java => src/main/java/org/apache/linkis/cli/application/interactor/command/CliCmdType.java} (78%) rename linkis-computation-governance/linkis-client/linkis-cli/{linkis-cli-core/src/main/java/org/apache/linkis/cli/core => src/main/java/org/apache/linkis/cli/application}/interactor/command/CmdTemplateFactory.java (83%) rename linkis-computation-governance/linkis-client/linkis-cli/{linkis-cli-core/src/main/java/org/apache/linkis/cli/core => src/main/java/org/apache/linkis/cli/application}/interactor/command/SpecialMap.java (94%) rename linkis-computation-governance/linkis-client/linkis-cli/{linkis-cli-core/src/main/java/org/apache/linkis/cli/core => src/main/java/org/apache/linkis/cli/application}/interactor/command/fitter/AbstractFitter.java (89%) rename linkis-computation-governance/linkis-client/linkis-cli/{linkis-cli-core/src/main/java/org/apache/linkis/cli/core => src/main/java/org/apache/linkis/cli/application}/interactor/command/fitter/Fitter.java (80%) rename linkis-computation-governance/linkis-client/linkis-cli/{linkis-cli-core/src/main/java/org/apache/linkis/cli/core => src/main/java/org/apache/linkis/cli/application}/interactor/command/fitter/FitterResult.java (90%) rename linkis-computation-governance/linkis-client/linkis-cli/{linkis-cli-core/src/main/java/org/apache/linkis/cli/core => src/main/java/org/apache/linkis/cli/application}/interactor/command/fitter/FitterUtils.java (93%) rename linkis-computation-governance/linkis-client/linkis-cli/{linkis-cli-core/src/main/java/org/apache/linkis/cli/core => src/main/java/org/apache/linkis/cli/application}/interactor/command/fitter/SingleTplFitter.java (76%) rename linkis-computation-governance/linkis-client/linkis-cli/{linkis-cli-core/src/main/java/org/apache/linkis/cli/core => src/main/java/org/apache/linkis/cli/application}/interactor/command/parser/AbstarctParser.java (85%) rename linkis-computation-governance/linkis-client/linkis-cli/{linkis-cli-core/src/main/java/org/apache/linkis/cli/core => src/main/java/org/apache/linkis/cli/application}/interactor/command/parser/Parser.java (83%) rename linkis-computation-governance/linkis-client/linkis-cli/{linkis-cli-core/src/main/java/org/apache/linkis/cli/core => src/main/java/org/apache/linkis/cli/application}/interactor/command/parser/SingleCmdParser.java (69%) rename linkis-computation-governance/linkis-client/linkis-cli/{linkis-cli-core/src/main/java/org/apache/linkis/cli/core => src/main/java/org/apache/linkis/cli/application}/interactor/command/parser/result/ParseResult.java (88%) rename linkis-computation-governance/linkis-client/linkis-cli/{linkis-cli-core/src/main/java/org/apache/linkis/cli/core => src/main/java/org/apache/linkis/cli/application}/interactor/command/parser/transformer/ParamKeyMapper.java (88%) rename linkis-computation-governance/linkis-client/linkis-cli/{linkis-cli-core/src/main/java/org/apache/linkis/cli/core => src/main/java/org/apache/linkis/cli/application}/interactor/command/template/AbstractCmdTemplate.java (91%) rename linkis-computation-governance/linkis-client/linkis-cli/{linkis-cli-application => }/src/main/java/org/apache/linkis/cli/application/interactor/command/template/UniversalCmdTemplate.java (75%) rename linkis-computation-governance/linkis-client/linkis-cli/{linkis-cli-core/src/main/java/org/apache/linkis/cli/core/utils => src/main/java/org/apache/linkis/cli/application/interactor/command/template}/converter/AbstractStringConverter.java (91%) rename linkis-computation-governance/linkis-client/linkis-cli/{linkis-cli-core/src/main/java/org/apache/linkis/cli/core/utils => src/main/java/org/apache/linkis/cli/application/interactor/command/template}/converter/Converter.java (91%) rename linkis-computation-governance/linkis-client/linkis-cli/{linkis-cli-core/src/main/java/org/apache/linkis/cli/core/utils => src/main/java/org/apache/linkis/cli/application/interactor/command/template}/converter/PredefinedStringConverters.java (92%) rename linkis-computation-governance/linkis-client/linkis-cli/{linkis-cli-core/src/main/java/org/apache/linkis/cli/core => src/main/java/org/apache/linkis/cli/application}/interactor/command/template/option/BaseOption.java (93%) rename linkis-computation-governance/linkis-client/linkis-cli/{linkis-cli-core/src/main/java/org/apache/linkis/cli/core => src/main/java/org/apache/linkis/cli/application}/interactor/command/template/option/Flag.java (91%) rename linkis-computation-governance/linkis-client/linkis-cli/{linkis-cli-core/src/main/java/org/apache/linkis/cli/core => src/main/java/org/apache/linkis/cli/application}/interactor/command/template/option/MapOption.java (93%) rename linkis-computation-governance/linkis-client/linkis-cli/{linkis-cli-core/src/main/java/org/apache/linkis/cli/core => src/main/java/org/apache/linkis/cli/application}/interactor/command/template/option/Parameter.java (93%) rename linkis-computation-governance/linkis-client/linkis-cli/{linkis-cli-core/src/main/java/org/apache/linkis/cli/core => src/main/java/org/apache/linkis/cli/application}/interactor/command/template/option/SpecialMapOption.java (90%) rename linkis-computation-governance/linkis-client/linkis-cli/{linkis-cli-core/src/main/java/org/apache/linkis/cli/core => src/main/java/org/apache/linkis/cli/application}/interactor/command/template/option/StdOption.java (92%) create mode 100644 linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/context/CliCtxImpl.java create mode 100644 linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/job/common/JobKiller.java rename linkis-computation-governance/linkis-client/linkis-cli/{linkis-cli-application/src/main/java/org/apache/linkis/cli/application/interactor/job/builder/ProcessKeyUtils.java => src/main/java/org/apache/linkis/cli/application/interactor/job/common/KeyParser.java} (85%) create mode 100644 linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/job/common/KillResult.java rename linkis-computation-governance/linkis-client/linkis-cli/{linkis-cli-application/src/main/java/org/apache/linkis/cli/application/interactor/job => src/main/java/org/apache/linkis/cli/application/interactor/job/common}/LinkisJobStatus.java (97%) create mode 100644 linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/job/common/LogData.java create mode 100644 linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/job/common/LogRetriever.java create mode 100644 linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/job/common/ResultData.java create mode 100644 linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/job/common/ResultRetriever.java rename linkis-computation-governance/linkis-client/linkis-cli/{linkis-cli-application/src/main/java/org/apache/linkis/cli/application/interactor/job/data/LinkisResultSet.java => src/main/java/org/apache/linkis/cli/application/interactor/job/common/ResultSet.java} (88%) create mode 100644 linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/job/help/HelpJob.java create mode 100644 linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/job/interactive/InteractiveJob.java rename linkis-computation-governance/linkis-client/linkis-cli/{linkis-cli-application/src/main/java/org/apache/linkis/cli/application/interactor/job/desc/LinkisSubmitDesc.java => src/main/java/org/apache/linkis/cli/application/interactor/job/interactive/InteractiveJobDesc.java} (93%) create mode 100644 linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/job/interactive/InteractiveJobDescBuilder.java create mode 100644 linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/job/interactive/InteractiveJobResult.java rename linkis-computation-governance/linkis-client/linkis-cli/{linkis-cli-application/src/main/java/org/apache/linkis/cli/application/interactor/job/desc/LinkisJobManDesc.java => src/main/java/org/apache/linkis/cli/application/interactor/job/jobcmd/JobCmdDesc.java} (82%) create mode 100644 linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/job/jobcmd/JobCmdDescBuilder.java create mode 100644 linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/job/jobcmd/JobCmdJob.java create mode 100644 linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/job/jobcmd/JobCmdJobResult.java rename linkis-computation-governance/linkis-client/linkis-cli/{linkis-cli-application/src/main/java/org/apache/linkis/cli/application/interactor/job/subtype/LinkisManSubType.java => src/main/java/org/apache/linkis/cli/application/interactor/job/jobcmd/JobCmdSubType.java} (81%) create mode 100644 linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/job/once/LinkisOnceJob.java create mode 100644 linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/job/once/OnceJobResult.java create mode 100644 linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/job/version/VersionJob.java rename linkis-computation-governance/linkis-client/linkis-cli/{linkis-cli-core/src/main/java/org/apache/linkis/cli/core/operator/JobOperatorBuilder.java => src/main/java/org/apache/linkis/cli/application/interactor/job/version/VersionJobResult.java} (55%) rename linkis-computation-governance/linkis-client/linkis-cli/{linkis-cli-common/src/main/java/org/apache/linkis/cli/common/entity => src/main/java/org/apache/linkis/cli/application/interactor}/properties/ClientProperties.java (95%) rename linkis-computation-governance/linkis-client/linkis-cli/{linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/properties/StdPropsLoader.java => src/main/java/org/apache/linkis/cli/application/interactor/properties/PropertiesLoader.java} (82%) rename linkis-computation-governance/linkis-client/linkis-cli/{linkis-cli-core/src/main/java/org/apache/linkis/cli/core => src/main/java/org/apache/linkis/cli/application}/interactor/properties/PropsFilesScanner.java (81%) rename linkis-computation-governance/linkis-client/linkis-cli/{linkis-cli-core/src/main/java/org/apache/linkis/cli/core => src/main/java/org/apache/linkis/cli/application}/interactor/properties/reader/PropertiesReader.java (93%) rename linkis-computation-governance/linkis-client/linkis-cli/{linkis-cli-core/src/main/java/org/apache/linkis/cli/core => src/main/java/org/apache/linkis/cli/application}/interactor/properties/reader/PropsFileReader.java (90%) rename linkis-computation-governance/linkis-client/linkis-cli/{linkis-cli-application/src/main/java/org/apache/linkis/cli/application/interactor/validate/LinkisSubmitValidator.java => src/main/java/org/apache/linkis/cli/application/interactor/validate/InteractiveDescValidator.java} (76%) rename linkis-computation-governance/linkis-client/linkis-cli/{linkis-cli-application/src/main/java/org/apache/linkis/cli/application/interactor/validate/LinkisManageValidator.java => src/main/java/org/apache/linkis/cli/application/interactor/validate/JobCmdDescValidator.java} (60%) rename linkis-computation-governance/linkis-client/linkis-cli/{linkis-cli-application/src/main/java/org/apache/linkis/cli/application/interactor/validate/LinkisOnceSubmitValidator.java => src/main/java/org/apache/linkis/cli/application/interactor/validate/OnceDescValidator.java} (77%) rename linkis-computation-governance/linkis-client/linkis-cli/{linkis-cli-core/src/main/java/org/apache/linkis/cli/core => src/main/java/org/apache/linkis/cli/application}/interactor/validate/ParamValidator.java (75%) rename linkis-computation-governance/linkis-client/linkis-cli/{linkis-cli-core/src/main/java/org/apache/linkis/cli/core => src/main/java/org/apache/linkis/cli/application}/interactor/validate/ParsedTplValidator.java (69%) rename linkis-computation-governance/linkis-client/linkis-cli/{linkis-cli-application => }/src/main/java/org/apache/linkis/cli/application/interactor/validate/UJESContextValidator.java (75%) rename linkis-computation-governance/linkis-client/linkis-cli/{linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/var/StdVarAccess.java => src/main/java/org/apache/linkis/cli/application/interactor/var/VarAccessImpl.java} (88%) rename linkis-computation-governance/linkis-client/linkis-cli/{linkis-cli-application/src/main/java/org/apache/linkis/cli/application/observer/event/TriggerEvent.java => src/main/java/org/apache/linkis/cli/application/observer/event/FetchResultEvent.java} (93%) rename linkis-computation-governance/linkis-client/linkis-cli/{linkis-cli-application => }/src/main/java/org/apache/linkis/cli/application/observer/event/LinkisClientEvent.java (100%) rename linkis-computation-governance/linkis-client/linkis-cli/{linkis-cli-application => }/src/main/java/org/apache/linkis/cli/application/observer/event/LogStartEvent.java (100%) rename linkis-computation-governance/linkis-client/linkis-cli/{linkis-cli-application => }/src/main/java/org/apache/linkis/cli/application/observer/event/SingleObserverEvent.java (100%) rename linkis-computation-governance/linkis-client/linkis-cli/{linkis-cli-application => }/src/main/java/org/apache/linkis/cli/application/observer/listener/LinkisClientListener.java (100%) rename linkis-computation-governance/linkis-client/linkis-cli/{linkis-cli-common/src/main/java/org/apache/linkis/cli/common/entity/validate/Validator.java => src/main/java/org/apache/linkis/cli/application/operator/JobOperBuilder.java} (76%) rename linkis-computation-governance/linkis-client/linkis-cli/{linkis-cli-application/src/test/java/org/apache/linkis/cli/application/operator/OperatorUtilsTest.java => src/main/java/org/apache/linkis/cli/application/operator/OperManager.java} (56%) rename linkis-computation-governance/linkis-client/linkis-cli/{linkis-cli-application => }/src/main/java/org/apache/linkis/cli/application/operator/once/LinkisNodeStatus.java (100%) rename linkis-computation-governance/linkis-client/linkis-cli/{linkis-cli-application => }/src/main/java/org/apache/linkis/cli/application/operator/once/OnceJobConstants.java (100%) rename linkis-computation-governance/linkis-client/linkis-cli/{linkis-cli-application/src/main/java/org/apache/linkis/cli/application/interactor/job/desc/LinkisOnceDesc.java => src/main/java/org/apache/linkis/cli/application/operator/once/OnceJobDesc.java} (82%) create mode 100644 linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/operator/once/OnceJobOper.java create mode 100644 linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/operator/once/OnceOperBuilder.java rename linkis-computation-governance/linkis-client/linkis-cli/{linkis-cli-application/src/main/java/org/apache/linkis/cli/application/operator/ujes/LinkisJobOperator.java => src/main/java/org/apache/linkis/cli/application/operator/ujes/LinkisJobOper.java} (75%) rename linkis-computation-governance/linkis-client/linkis-cli/{linkis-cli-application/src/main/java/org/apache/linkis/cli/application/operator/ujes/LinkisOperatorBuilder.java => src/main/java/org/apache/linkis/cli/application/operator/ujes/LinkisOperBuilder.java} (60%) rename linkis-computation-governance/linkis-client/linkis-cli/{linkis-cli-application => }/src/main/java/org/apache/linkis/cli/application/operator/ujes/LinkisOperResultAdapter.java (90%) rename linkis-computation-governance/linkis-client/linkis-cli/{linkis-cli-application => }/src/main/java/org/apache/linkis/cli/application/operator/ujes/UJESClientContext.java (100%) rename linkis-computation-governance/linkis-client/linkis-cli/{linkis-cli-application => }/src/main/java/org/apache/linkis/cli/application/operator/ujes/UJESClientFactory.java (79%) rename linkis-computation-governance/linkis-client/linkis-cli/{linkis-cli-application => }/src/main/java/org/apache/linkis/cli/application/operator/ujes/UJESConstants.java (100%) rename linkis-computation-governance/linkis-client/linkis-cli/{linkis-cli-application => }/src/main/java/org/apache/linkis/cli/application/operator/ujes/UJESResultAdapter.java (90%) rename linkis-computation-governance/linkis-client/linkis-cli/{linkis-cli-application => }/src/main/java/org/apache/linkis/cli/application/operator/ujes/result/OpenLogResult2.java (100%) rename linkis-computation-governance/linkis-client/linkis-cli/{linkis-cli-application => }/src/main/java/org/apache/linkis/cli/application/operator/ujes/result/ResultSetResult2.java (100%) rename linkis-computation-governance/linkis-client/linkis-cli/{linkis-cli-application => }/src/main/java/org/apache/linkis/cli/application/operator/ujes/result/UJESResult.java (100%) rename linkis-computation-governance/linkis-client/linkis-cli/{linkis-cli-core/src/main/java/org/apache/linkis/cli/core/present/HelpInfoPresenter.java => src/main/java/org/apache/linkis/cli/application/present/HelpPresenter.java} (65%) rename linkis-computation-governance/linkis-client/linkis-cli/{linkis-cli-core/src/main/java/org/apache/linkis/cli/core/present/PresentModeImpl.java => src/main/java/org/apache/linkis/cli/application/present/JobInfoPresenter.java} (63%) rename linkis-computation-governance/linkis-client/linkis-cli/{linkis-cli-application/src/main/java/org/apache/linkis/cli/application/present/LinkisLogPresenter.java => src/main/java/org/apache/linkis/cli/application/present/LogPresenter.java} (61%) create mode 100644 linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/present/ResultPresenter.java rename linkis-computation-governance/linkis-client/linkis-cli/{linkis-cli-core/src/main/java/org/apache/linkis/cli/core/present/display/PlainTextFileWriter.java => src/main/java/org/apache/linkis/cli/application/present/file/ResultFileWriter.java} (72%) rename linkis-computation-governance/linkis-client/linkis-cli/{linkis-cli-core/src/main/java/org/apache/linkis/cli/core => src/main/java/org/apache/linkis/cli/application}/present/model/HelpInfoModel.java (77%) create mode 100644 linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/present/model/LinkisJobInfoModel.java rename linkis-computation-governance/linkis-client/linkis-cli/{linkis-cli-application => }/src/main/java/org/apache/linkis/cli/application/present/model/LinkisLogModel.java (74%) rename linkis-computation-governance/linkis-client/linkis-cli/{linkis-cli-application => }/src/main/java/org/apache/linkis/cli/application/present/model/LinkisResultModel.java (63%) rename linkis-computation-governance/linkis-client/linkis-cli/{linkis-cli-application/src/main/java/org/apache/linkis/cli/application/utils/ExecutionUtils.java => src/main/java/org/apache/linkis/cli/application/utils/CliUtils.java} (65%) rename linkis-computation-governance/linkis-client/linkis-cli/{linkis-cli-core/src/main/java/org/apache/linkis/cli/core/utils/LogUtils.java => src/main/java/org/apache/linkis/cli/application/utils/LoggerManager.java} (93%) rename linkis-computation-governance/linkis-client/linkis-cli/{linkis-cli-core/src/main/java/org/apache/linkis/cli/core/utils/SchedulerUtils.java => src/main/java/org/apache/linkis/cli/application/utils/SchedulerManager.java} (94%) create mode 100644 linkis-computation-governance/linkis-client/linkis-cli/src/main/resources/version.properties rename linkis-computation-governance/linkis-client/linkis-cli/{linkis-cli-application => }/src/test/java/org/apache/linkis/cli/application/LinkisClientApplicationTest.java (88%) rename linkis-computation-governance/linkis-client/linkis-cli/{linkis-cli-application/src/test/java/org/apache/linkis/cli/application/constants/AppConstantsTest.java => src/test/java/org/apache/linkis/cli/application/constants/CliConstantsTest.java} (71%) rename linkis-computation-governance/linkis-client/linkis-cli/{linkis-cli-application/src/test/java/org/apache/linkis/cli/application/constants/AppKeysTest.java => src/test/java/org/apache/linkis/cli/application/constants/CliKeysTest.java} (73%) rename linkis-computation-governance/linkis-client/linkis-cli/{linkis-cli-application => }/src/test/java/org/apache/linkis/cli/application/constants/LinkisConstantsTest.java (100%) rename linkis-computation-governance/linkis-client/linkis-cli/{linkis-cli-application => }/src/test/java/org/apache/linkis/cli/application/constants/LinkisKeysTest.java (100%) rename linkis-computation-governance/linkis-client/linkis-cli/{linkis-cli-application => }/src/test/java/org/apache/linkis/cli/application/constants/TestConstants.java (100%) rename linkis-computation-governance/linkis-client/linkis-cli/{linkis-cli-application/src/test/java/org/apache/linkis/cli/application/interactor/command/LinkisCmdTypeTest.java => src/test/java/org/apache/linkis/cli/application/interactor/command/CliCmdTypeTest.java} (86%) rename linkis-computation-governance/linkis-client/linkis-cli/{linkis-cli-application => }/src/test/java/org/apache/linkis/cli/application/interactor/command/template/TestCmdType.java (96%) rename linkis-computation-governance/linkis-client/linkis-cli/{linkis-cli-application => }/src/test/java/org/apache/linkis/cli/application/interactor/command/template/TestParamMapper.java (78%) rename linkis-computation-governance/linkis-client/linkis-cli/{linkis-cli-application => }/src/test/java/org/apache/linkis/cli/application/interactor/command/template/TestSparkCmdTemplate.java (94%) rename linkis-computation-governance/linkis-client/linkis-cli/{linkis-cli-application => }/src/test/java/org/apache/linkis/cli/application/interactor/job/LinkisJobStatusTest.java (95%) rename linkis-computation-governance/linkis-client/linkis-cli/{linkis-cli-application/src/test/java/org/apache/linkis/cli/application/interactor/job/builder/ProcessKeyUtilsTest.java => src/test/java/org/apache/linkis/cli/application/interactor/job/builder/KeyParserTest.java} (81%) rename linkis-computation-governance/linkis-client/linkis-cli/{linkis-cli-application/src/test/java/org/apache/linkis/cli/application/interactor/job/subtype/LinkisManSubTypeTest.java => src/test/java/org/apache/linkis/cli/application/interactor/job/subtype/JobCmdSubTypeTest.java} (75%) rename linkis-computation-governance/linkis-client/linkis-cli/{linkis-cli-application => }/src/test/java/org/apache/linkis/cli/application/operator/once/OnceJobConstantsTest.java (100%) rename linkis-computation-governance/linkis-client/linkis-cli/{linkis-cli-application => }/src/test/java/org/apache/linkis/cli/application/operator/ujes/UJESConstantsTest.java (100%) rename linkis-computation-governance/linkis-client/linkis-cli/{linkis-cli-application => }/src/test/java/org/apache/linkis/cli/application/utils/UtilsTest.java (95%) rename linkis-computation-governance/linkis-client/linkis-cli/{linkis-cli-application => }/src/test/resources/conf/linkis-cli.properties (100%) rename linkis-computation-governance/linkis-client/linkis-cli/{linkis-cli-application => }/src/test/resources/conf/user.properties (100%) rename linkis-computation-governance/linkis-client/linkis-cli/{linkis-cli-application => }/src/test/resources/linkis-cli.properties (100%) rename linkis-computation-governance/linkis-client/linkis-cli/{linkis-cli-application => }/src/test/resources/log4j2.xml (100%) rename linkis-computation-governance/linkis-client/linkis-cli/{linkis-cli-application => }/src/test/resources/testScala.scala (100%) diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/pom.xml b/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/pom.xml deleted file mode 100644 index 980141f9a3..0000000000 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/pom.xml +++ /dev/null @@ -1,87 +0,0 @@ - - - - 4.0.0 - - - org.apache.linkis - linkis-cli - ${revision} - - linkis-cli-application - jar - - - - org.apache.linkis - linkis-cli-common - ${project.version} - - - org.apache.linkis - linkis-cli-core - ${project.version} - - - org.apache.linkis - linkis-computation-client - ${project.version} - - - org.apache.linkis - linkis-gateway-httpclient-support - ${project.version} - - - - - - - - org.apache.maven.plugins - maven-assembly-plugin - false - - false - out - false - false - - /src/main/assembly/distribution.xml - - - - - make-assembly - - single - - package - - - /src/main/assembly/distribution.xml - - - - - - - - - - diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/LinkisClientApplication.java b/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/LinkisClientApplication.java deleted file mode 100644 index 0c6fc1ec73..0000000000 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/LinkisClientApplication.java +++ /dev/null @@ -1,341 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.cli.application; - -import org.apache.linkis.cli.application.constants.AppConstants; -import org.apache.linkis.cli.application.constants.AppKeys; -import org.apache.linkis.cli.application.data.FinishedData; -import org.apache.linkis.cli.application.data.PreparedData; -import org.apache.linkis.cli.application.data.ProcessedData; -import org.apache.linkis.cli.application.interactor.command.LinkisCmdType; -import org.apache.linkis.cli.application.interactor.command.template.UniversalCmdTemplate; -import org.apache.linkis.cli.application.operator.ujes.LinkisOperatorBuilder; -import org.apache.linkis.cli.application.suite.ExecutionSuite; -import org.apache.linkis.cli.application.suite.ExecutionSuiteFactory; -import org.apache.linkis.cli.application.utils.Utils; -import org.apache.linkis.cli.common.entity.command.CmdTemplate; -import org.apache.linkis.cli.common.entity.command.Params; -import org.apache.linkis.cli.common.entity.execution.Execution; -import org.apache.linkis.cli.common.entity.job.Job; -import org.apache.linkis.cli.common.entity.properties.ClientProperties; -import org.apache.linkis.cli.common.entity.result.ExecutionResult; -import org.apache.linkis.cli.common.entity.result.ResultHandler; -import org.apache.linkis.cli.common.entity.var.VarAccess; -import org.apache.linkis.cli.common.exception.LinkisClientRuntimeException; -import org.apache.linkis.cli.common.exception.error.ErrorLevel; -import org.apache.linkis.cli.common.exception.handler.ExceptionHandler; -import org.apache.linkis.cli.core.constants.CommonConstants; -import org.apache.linkis.cli.core.exception.CommandException; -import org.apache.linkis.cli.core.exception.PropsException; -import org.apache.linkis.cli.core.exception.error.CommonErrMsg; -import org.apache.linkis.cli.core.exception.handler.CommandExceptionHandler; -import org.apache.linkis.cli.core.exception.handler.DefaultExceptionHandler; -import org.apache.linkis.cli.core.interactor.command.CmdTemplateFactory; -import org.apache.linkis.cli.core.interactor.command.fitter.SingleTplFitter; -import org.apache.linkis.cli.core.interactor.command.parser.Parser; -import org.apache.linkis.cli.core.interactor.command.parser.SingleCmdParser; -import org.apache.linkis.cli.core.interactor.command.parser.result.ParseResult; -import org.apache.linkis.cli.core.interactor.properties.PropertiesLoader; -import org.apache.linkis.cli.core.interactor.properties.PropsFilesScanner; -import org.apache.linkis.cli.core.interactor.properties.StdPropsLoader; -import org.apache.linkis.cli.core.interactor.properties.reader.PropertiesReader; -import org.apache.linkis.cli.core.interactor.properties.reader.PropsFileReader; -import org.apache.linkis.cli.core.interactor.properties.reader.SysEnvReader; -import org.apache.linkis.cli.core.interactor.properties.reader.SysPropsReader; -import org.apache.linkis.cli.core.interactor.result.DefaultResultHandler; -import org.apache.linkis.cli.core.interactor.result.ExecutionResultImpl; -import org.apache.linkis.cli.core.interactor.result.ExecutionStatusEnum; -import org.apache.linkis.cli.core.interactor.validate.ParsedTplValidator; -import org.apache.linkis.cli.core.interactor.var.StdVarAccess; -import org.apache.linkis.cli.core.interactor.var.SysVarAccess; -import org.apache.linkis.cli.core.operator.JobOperatorBuilder; -import org.apache.linkis.cli.core.operator.JobOperatorFactory; -import org.apache.linkis.cli.core.present.PresentModeImpl; -import org.apache.linkis.cli.core.present.display.DisplayOperFactory; -import org.apache.linkis.cli.core.present.display.PlainTextFileWriter; -import org.apache.linkis.cli.core.present.display.StdOutWriter; -import org.apache.linkis.cli.core.utils.LogUtils; -import org.apache.linkis.cli.core.utils.SchedulerUtils; - -import org.apache.commons.lang3.StringUtils; - -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public class LinkisClientApplication { - private static Logger logger = LoggerFactory.getLogger(LinkisClientApplication.class); - - /** - * generate Templates load env variables TODO: load version info - * - * @return PreparedData - */ - private static PreparedData prepare() throws LinkisClientRuntimeException { - /* - generate template - */ - CmdTemplate template = new UniversalCmdTemplate(); - CmdTemplateFactory.register(template); - /* - load env variables - */ - Map propertiesMap = new HashMap<>(); - PropertiesLoader loader = - new StdPropsLoader() - .addPropertiesReader(new SysPropsReader()) - .addPropertiesReader(new SysEnvReader()); - for (ClientProperties properties : loader.loadProperties()) { - propertiesMap.put(properties.getPropsId(), properties); - } - - return new PreparedData(propertiesMap); - } - - /** - * parse user input load user config load default config check if all inputs are ok - * - * @param args user input arguments - * @return ProcessedData - */ - private static ProcessedData processInput(String[] args, PreparedData preparedData) - throws Exception { - - if (preparedData == null) { - return null; - } - - /* - user input - */ - CmdTemplate template = CmdTemplateFactory.getTemplateCopy(LinkisCmdType.UNIVERSAL); - Parser parser = - new SingleCmdParser() - .setMapper(null) - .setTemplate(template) - .setFitter(new SingleTplFitter()); - - ParseResult result = parser.parse(args); - - ParsedTplValidator parsedTplValidator = new ParsedTplValidator(); - parsedTplValidator.doValidation(result.getParsedTemplate()); - - Params params = result.getParams(); - logger.debug("==========params============\n" + Utils.GSON.toJson(params)); - - /* - VarAccess for sys_prop, sys_env - */ - Map propertiesMap = preparedData.getPropertiesMap(); - VarAccess sysVarAccess = - new SysVarAccess() - .setSysProp(propertiesMap.get(CommonConstants.SYSTEM_PROPERTIES_IDENTIFIER)) - .setSysEnv(propertiesMap.get(CommonConstants.SYSTEM_ENV_IDENTIFIER)); - logger.debug("==========sys_var============\n" + Utils.GSON.toJson(sysVarAccess)); - - LogUtils.getInformationLogger() - .info( - "LogFile path: " - + sysVarAccess.getVar(String.class, AppKeys.LOG_PATH_KEY) - + "/" - + sysVarAccess.getVar(String.class, AppKeys.LOG_FILE_KEY)); - /* - default config, -Dconf.root & -Dconf.file specifies config path - */ - // scan config files given root path - String configPath = sysVarAccess.getVar(String.class, AppKeys.CLIENT_CONFIG_ROOT_KEY); - String defaultConfFileName = - sysVarAccess.getVarOrDefault( - String.class, AppKeys.DEFAULT_CONFIG_FILE_NAME_KEY, AppConstants.DEFAULT_CONFIG_NAME); - if (StringUtils.isBlank(configPath)) { - throw new PropsException( - "PRP0007", - ErrorLevel.ERROR, - CommonErrMsg.PropsLoaderErr, - "configuration root path specified by env variable: " - + AppKeys.CLIENT_CONFIG_ROOT_KEY - + " is empty."); - } - - List readersList = - new PropsFilesScanner().getPropsReaders(configPath); // +1 user config - /* - user defined config - */ - String userConfPath = null; - if (params.containsParam(AppKeys.LINKIS_CLIENT_USER_CONFIG)) { - userConfPath = - (String) params.getParamItemMap().get(AppKeys.LINKIS_CLIENT_USER_CONFIG).getValue(); - } - if (StringUtils.isNotBlank(userConfPath)) { - PropertiesReader reader = - new PropsFileReader() - .setPropsId(AppKeys.LINKIS_CLIENT_USER_CONFIG) - .setPropsPath(userConfPath); - readersList.add(reader); - } else { - LogUtils.getInformationLogger() - .info("User does not provide usr-configuration file. Will use default config"); - } - /* - load properties - */ - PropertiesLoader loader = - new StdPropsLoader() - .addPropertiesReaders(readersList.toArray(new PropertiesReader[readersList.size()])); - ClientProperties[] loaderResult = loader.loadProperties(); - for (ClientProperties properties : loaderResult) { - if (StringUtils.equals(properties.getPropsId(), AppKeys.LINKIS_CLIENT_USER_CONFIG)) { - for (Map.Entry prop : properties.entrySet()) { - if (StringUtils.startsWith( - (String) prop.getKey(), AppKeys.LINKIS_CLIENT_NONCUSTOMIZABLE)) { - throw new PropsException( - "PRP0007", - ErrorLevel.ERROR, - CommonErrMsg.PropsLoaderErr, - "User cannot specify non-customizable configuration: " + prop.getKey()); - } - } - } - propertiesMap.put(properties.getPropsId(), properties); - } - - /* - VarAccess for cmd, config - */ - VarAccess stdVarAccess = - new StdVarAccess() - .setCmdParams(params) - .setUserConf(propertiesMap.get(AppKeys.LINKIS_CLIENT_USER_CONFIG)) - .setDefaultConf(propertiesMap.get(defaultConfFileName)) - .init(); - logger.info("==========std_var============\n" + Utils.GSON.toJson(stdVarAccess)); - - /* - Prepare operator for accessing linkis - */ - JobOperatorBuilder builder = - new LinkisOperatorBuilder().setStdVarAccess(stdVarAccess).setSysVarAccess(sysVarAccess); - - JobOperatorFactory.register(AppKeys.REUSABLE_UJES_CLIENT, builder); - /* - Prepare DisplayOperator - */ - DisplayOperFactory.register(PresentModeImpl.STDOUT, new StdOutWriter()); - DisplayOperFactory.register(PresentModeImpl.TEXT_FILE, new PlainTextFileWriter()); - - return new ProcessedData( - AppConstants.DUMMY_CID, params.getCmdType(), stdVarAccess, sysVarAccess); - } - - /** - * submit job display result - * - * @return FinishedData - */ - private static FinishedData exec(ProcessedData data) throws Exception { - if (data == null) { - return null; - } - - ExecutionSuite suite = - ExecutionSuiteFactory.getSuite( - data.getCmdType(), data.getStdVarAccess(), data.getSysVarAccess()); - - /* - Get everything - */ - Map jobs = suite.getJobs(); - ResultHandler[] resultHandlers = suite.getResultHandlers(); - Execution execution = suite.getExecution(); - - /* - execute - */ - final Map jobsToKill = jobs; - Thread hook = new Thread(() -> execution.terminate(jobsToKill)); - if (jobsToKill != null && jobsToKill.size() != 0) { - Runtime.getRuntime().addShutdownHook(hook); - } - ExecutionResult result = execution.execute(jobs); - - Runtime.getRuntime().removeShutdownHook(hook); - - return new FinishedData(result, resultHandlers); - } - - public static void main(String[] args) { - - ExceptionHandler handler = new DefaultExceptionHandler(); - ProcessedData processedData = null; - FinishedData finishedData = null; - ExecutionResult executionResult = new ExecutionResultImpl(null, ExecutionStatusEnum.UNDEFINED); - PreparedData preparedData = null; - - try { - preparedData = prepare(); - } catch (Exception e) { - handler.handle(e); - executionResult.setExecutionStatus(ExecutionStatusEnum.FAILED); - } - - try { - processedData = processInput(args, preparedData); - } catch (CommandException ce) { - new CommandExceptionHandler().handle(ce); - executionResult.setExecutionStatus(ExecutionStatusEnum.FAILED); - } catch (Exception e) { - handler.handle(e); - executionResult.setExecutionStatus(ExecutionStatusEnum.FAILED); - } - - try { - finishedData = exec(processedData); - } catch (Exception e) { - handler.handle(e); - executionResult.setExecutionStatus(ExecutionStatusEnum.FAILED); - } - - if (finishedData != null) { - executionResult = finishedData.getExecutionResult(); - if (executionResult == null) { - executionResult = new ExecutionResultImpl(null, ExecutionStatusEnum.UNDEFINED); - } - if (executionResult.getException() != null) { - handler.handle(executionResult.getException()); - new DefaultResultHandler().process(executionResult); - } else { - if (finishedData.getResultHandlers() != null) { - for (ResultHandler resultHandler : finishedData.getResultHandlers()) { - if (resultHandler != null) { - resultHandler.process(executionResult); - } - } - } - } - } else { - executionResult.setExecutionStatus(ExecutionStatusEnum.FAILED); - new DefaultResultHandler().process(executionResult); - } - - SchedulerUtils.shutDown(); - } -} diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/data/FinishedData.java b/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/data/FinishedData.java deleted file mode 100644 index 4438cf301b..0000000000 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/data/FinishedData.java +++ /dev/null @@ -1,47 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.cli.application.data; - -import org.apache.linkis.cli.common.entity.result.ExecutionResult; -import org.apache.linkis.cli.common.entity.result.ResultHandler; - -public class FinishedData { - ExecutionResult executionResult; - ResultHandler[] resultHandlers; - - public FinishedData(ExecutionResult executionResult, ResultHandler[] resultHandlers) { - this.executionResult = executionResult; - this.resultHandlers = resultHandlers; - } - - public ExecutionResult getExecutionResult() { - return executionResult; - } - - public void setExecutionResult(ExecutionResult executionResult) { - this.executionResult = executionResult; - } - - public ResultHandler[] getResultHandlers() { - return resultHandlers; - } - - public void setResultHandlers(ResultHandler[] resultHandlers) { - this.resultHandlers = resultHandlers; - } -} diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/data/ProcessedData.java b/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/data/ProcessedData.java deleted file mode 100644 index a69b7c4b63..0000000000 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/data/ProcessedData.java +++ /dev/null @@ -1,52 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.cli.application.data; - -import org.apache.linkis.cli.common.entity.command.CmdType; -import org.apache.linkis.cli.common.entity.var.VarAccess; - -public class ProcessedData { - String cid; - CmdType cmdType; - VarAccess stdVarAccess; - VarAccess sysVarAccess; - - public ProcessedData( - String cid, CmdType cmdType, VarAccess stdVarAccess, VarAccess sysVarAccess) { - this.cid = cid; - this.cmdType = cmdType; - this.stdVarAccess = stdVarAccess; - this.sysVarAccess = sysVarAccess; - } - - public String getCid() { - return cid; - } - - public CmdType getCmdType() { - return cmdType; - } - - public VarAccess getStdVarAccess() { - return stdVarAccess; - } - - public VarAccess getSysVarAccess() { - return sysVarAccess; - } -} diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/interactor/job/LinkisJob.java b/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/interactor/job/LinkisJob.java deleted file mode 100644 index a32521c20e..0000000000 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/interactor/job/LinkisJob.java +++ /dev/null @@ -1,26 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.cli.application.interactor.job; - -import org.apache.linkis.cli.application.interactor.job.data.LinkisJobData; -import org.apache.linkis.cli.core.interactor.job.AbstractJob; - -public abstract class LinkisJob extends AbstractJob { - @Override - public abstract LinkisJobData getJobData(); -} diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/interactor/job/LinkisManageJob.java b/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/interactor/job/LinkisManageJob.java deleted file mode 100644 index 7dbcaca88d..0000000000 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/interactor/job/LinkisManageJob.java +++ /dev/null @@ -1,522 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.cli.application.interactor.job; - -import org.apache.linkis.cli.application.constants.AppConstants; -import org.apache.linkis.cli.application.interactor.job.data.LinkisJobData; -import org.apache.linkis.cli.application.interactor.job.data.LinkisLogData; -import org.apache.linkis.cli.application.interactor.job.data.LinkisResultData; -import org.apache.linkis.cli.application.interactor.job.desc.LinkisJobManDesc; -import org.apache.linkis.cli.application.interactor.job.subtype.LinkisManSubType; -import org.apache.linkis.cli.application.operator.ujes.LinkisJobOperator; -import org.apache.linkis.cli.application.utils.Utils; -import org.apache.linkis.cli.common.entity.job.JobData; -import org.apache.linkis.cli.common.entity.operator.JobOperator; -import org.apache.linkis.cli.common.exception.LinkisClientRuntimeException; -import org.apache.linkis.cli.common.exception.error.ErrorLevel; -import org.apache.linkis.cli.core.constants.CommonConstants; -import org.apache.linkis.cli.core.exception.LinkisClientExecutionException; -import org.apache.linkis.cli.core.exception.error.CommonErrMsg; -import org.apache.linkis.cli.core.interactor.job.*; -import org.apache.linkis.cli.core.utils.CommonUtils; -import org.apache.linkis.cli.core.utils.LogUtils; -import org.apache.linkis.cli.core.utils.SchedulerUtils; - -import org.apache.commons.lang3.StringUtils; - -import java.text.MessageFormat; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public class LinkisManageJob extends LinkisJob - implements ManagableBackendJob, TerminatableJob, LogAccessibleJob, ResultAccessibleJob { - private static final Logger logger = LoggerFactory.getLogger(LinkisManageJob.class); - - private LinkisJobManDesc jobDesc; - private LinkisJobData data; - private TerminateToken terminateToken = new TerminateToken(); - - @Override - public LinkisJobOperator getJobOperator() { - if (!(super.getJobOperator() instanceof LinkisJobOperator)) { - throw new LinkisClientExecutionException( - "EXE0003", - ErrorLevel.ERROR, - CommonErrMsg.ExecutionInitErr, - "JobOperator of LinkisManageJob should be instance of LinkisJobOperator"); - } - return (LinkisJobOperator) super.getJobOperator(); - } - - @Override - public void setOperator(JobOperator operator) { - if (!(operator instanceof LinkisJobOperator)) { - throw new LinkisClientExecutionException( - "EXE0003", - ErrorLevel.ERROR, - CommonErrMsg.ExecutionInitErr, - "JobOperator of LinkisManageJob should be instance of LinkisJobOperator"); - } - super.setOperator(operator); - } - - @Override - public LinkisJobManDesc getJobDesc() { - return jobDesc; - } - - public void setJobDesc(LinkisJobManDesc jobDesc) { - this.jobDesc = jobDesc; - } - - @Override - public LinkisJobData getJobData() { - return data; - } - - public void setJobData(LinkisJobData data) { - this.data = data; - } - - @Override - public boolean isSuccess() { - return data.isSuccess(); - } - - @Override - public void doManage() throws LinkisClientRuntimeException { - LinkisManSubType subType = (LinkisManSubType) getSubType(); - if (!(subType instanceof LinkisManSubType)) { - throw new LinkisClientExecutionException( - "EXE0030", - ErrorLevel.ERROR, - CommonErrMsg.ExecutionErr, - "JobSubType is not instance of JobManSubType"); - } - switch (subType) { - case STATUS: - try { - data.updateByOperResult( - getJobOperator().queryJobInfo(jobDesc.getUser(), jobDesc.getJobID())); - if (data.getJobStatus() != null) { - data.setSuccess(true); - } - } catch (Exception e) { - data.setSuccess(false); - data.setException(e); - } - break; - // case JOB_DESC: - // result = jobManagableBackendExecutor.queryJobDesc(job); - // break; - case LOG: - try { - ((LinkisLogData) data).setIncLogMode(false); - startRetrieveLog(); - waitLogFin(); - data.setSuccess(true); - } catch (Exception e) { - data.setSuccess(false); - data.setException(e); - } - break; - case RESULT: - try { - startRetrieveResult(); - data.setSuccess(true); - } catch (LinkisClientExecutionException e) { - if (e.getCode().equals("EXE0037")) { - ((LinkisResultData) data).sendResultFin(); // inform listener to stop - data.setSuccess(true); - } else { - data.setSuccess(false); - data.setException(e); - } - LogUtils.getInformationLogger().warn(e.getMessage()); - } catch (Exception e) { - data.setSuccess(false); - data.setException(e); - LogUtils.getInformationLogger().warn(e.getMessage()); - } - break; - // case LIST: - // resultData = jobManExecutor.queryJobList(job); - // break; - case KILL: - doKill(); - break; - default: - throw new LinkisClientExecutionException( - "EXE0002", - ErrorLevel.ERROR, - CommonErrMsg.ExecutionErr, - "JobSubType + \"" + subType + "\" is not supported"); - } - } - - @Override - public void startRetrieveLog() { - if (jobDesc.getUser() == null || jobDesc.getJobID() == null) { - throw new LinkisClientExecutionException( - "EXE0036", ErrorLevel.ERROR, CommonErrMsg.ExecutionErr, "user or jobID is null"); - } - data.updateByOperResult(getJobOperator().queryJobInfo(jobDesc.getUser(), jobDesc.getJobID())); - startRetrieveLogInternal(data); - } - - public void waitLogFin() { - if (!(data instanceof LinkisLogData)) { - return; - } - int retry = 0; - int MAX_RETRY = 300; // wait for 10 minutes after job finish - while (retry++ < MAX_RETRY) { - if (((LinkisLogData) data).logFinReceived()) { - return; - } - CommonUtils.doSleepQuietly(CommonConstants.JOB_QUERY_SLEEP_MILLS); - } - } - - public void startRetrieveLogInternal(JobData jobData) { - if (!(jobData instanceof LinkisLogData)) { - throw new LinkisClientExecutionException( - "EXE0034", ErrorLevel.ERROR, CommonErrMsg.ExecutionErr, "JobData is not LinkisLogData"); - } - if (jobData.getUser() == null || jobData.getJobID() == null) { - throw new LinkisClientExecutionException( - "EXE0036", ErrorLevel.ERROR, CommonErrMsg.ExecutionErr, "user or jobID is null"); - } - LinkisLogData logData = (LinkisLogData) jobData; - if (logData.getJobStatus() != null) { - try { - Thread logConsumer = new Thread(() -> logData.notifyLogListener(), "Log-Consumer"); - Thread logRetriever = new Thread(() -> queryLogLoop(logData), "Log-Retriever"); - SchedulerUtils.getCachedThreadPoolExecutor().execute(logRetriever); - SchedulerUtils.getCachedThreadPoolExecutor().execute(logConsumer); - } catch (Exception e) { - logger.warn("Failed to retrieve log", e); - } - } - } - - public void queryLogLoop(LinkisLogData data) { - int curLogIdx; - int nextLogIdx; - boolean hasNext = true; - int retryCnt = 0; - final int MAX_RETRY = 12; // continues fails for 90s, then exit thread - try { - while (hasNext) { - curLogIdx = data.getNextLogLineIdx() == null ? 0 : data.getNextLogLineIdx(); - try { - data.updateByOperResult(getJobOperator().queryJobInfo(data.getUser(), data.getJobID())); - queryJobLogFromLine(data, curLogIdx); - } catch (Exception e) { - logger.error("Cannot get inc-log:", e); - // and yes sometimes server may not be able to prepare persisted-log - retryCnt++; - if (retryCnt >= MAX_RETRY) { - logger.error( - "Continuously failing to query inc-log for " - + MAX_RETRY * (MAX_RETRY + 2) * 500 / 1000 - + "s. Will no longer try to query log", - e); - break; - } - Utils.doSleepQuietly(500l + 500l * retryCnt); // maybe server problem. sleep longer - continue; - } - retryCnt = 0; - nextLogIdx = data.getNextLogLineIdx() == null ? curLogIdx : data.getNextLogLineIdx(); - if (data.isIncLogMode()) { - hasNext = data.hasNextLogLine() == null ? curLogIdx < nextLogIdx : data.hasNextLogLine(); - } else { - hasNext = curLogIdx < nextLogIdx; - } - if (curLogIdx >= nextLogIdx) { - String msg = - MessageFormat.format( - "Job is still running, status={0}, progress={1}", - data.getJobStatus(), String.valueOf(data.getJobProgress() * 100) + "%"); - logger.info(msg); - } - Utils.doSleepQuietly(AppConstants.JOB_QUERY_SLEEP_MILLS); - } - } catch (Exception e) { - logger.error("Something goes wrong. Job Log may be incomplete", e); - } finally { - data.sendLogFin(); - } - } - - private void queryJobLogFromLine(LinkisLogData data, int fromLine) - throws LinkisClientRuntimeException { - if (!data.getJobStatus().isJobFinishedState()) { - try { - data.updateByOperResult( - getJobOperator() - .queryRunTimeLogFromLine( - data.getUser(), data.getJobID(), data.getExecID(), fromLine)); - } catch (Exception e) { - // job is finished while we start query log(but request is not send). - // then probably server cache is gone and we got a exception here. - // however we cannot know if this happens based on the exception message - logger.warn( - "Caught exception when querying runtime-log. Probably server-side has close stream. Will try openLog api if Job is completed.", - e); - if (data.getJobStatus().isJobFinishedState()) { - CommonUtils.doSleepQuietly(500l); - data.updateByOperResult( - getJobOperator() - .queryPersistedLogFromLine( - data.getUser(), data.getJobID(), data.getExecID(), fromLine)); - } - } - } else { - try { - data.updateByOperResult( - getJobOperator() - .queryPersistedLogFromLine( - data.getLogPath(), data.getUser(), data.getJobID(), fromLine)); - } catch (Exception e) { - logger.error("Cannot get persisted-inc-log:", e); - // and yes sometimes server may not be able to prepare persisted-log - throw e; - } - } - } - - @Override - public void startRetrieveResult() { - if (!(data instanceof LinkisResultData)) { - throw new LinkisClientExecutionException( - "EXE0034", - ErrorLevel.ERROR, - CommonErrMsg.ExecutionErr, - "JobData is not LinkisResultData"); - } - if (jobDesc.getUser() == null || jobDesc.getJobID() == null) { - throw new LinkisClientExecutionException( - "EXE0036", ErrorLevel.ERROR, CommonErrMsg.ExecutionErr, "user or jobID is null"); - } - data.updateByOperResult(getJobOperator().queryJobInfo(jobDesc.getUser(), jobDesc.getJobID())); - if (data.getJobStatus() == null) { - throw new LinkisClientExecutionException( - "EXE0038", ErrorLevel.ERROR, CommonErrMsg.ExecutionErr, "jobStatus is null"); - } - LinkisResultData resultData = (LinkisResultData) data; - if (!resultData.getJobStatus().isJobSuccess() - || StringUtils.isBlank(resultData.getResultLocation())) { - resultData.updateByOperResult( - getJobOperator().queryJobInfo(resultData.getUser(), resultData.getJobID())); - } - if (!resultData.getJobStatus().isJobSuccess()) { - // throw new LinkisClientExecutionException("EXE0035", ErrorLevel.ERROR, - // CommonErrMsg.ExecutionErr, "Job status is not success but \'" + - // resultData.getJobStatus() + "\'. Will not try to retrieve any Result"); - LogUtils.getInformationLogger() - .info( - "Job status is not success but \'" - + resultData.getJobStatus() - + "\'. Will not try to retrieve any Result"); - resultData.sendResultFin(); // inform listener to stop - return; - } - if (StringUtils.isBlank(resultData.getResultLocation())) { - throw new LinkisClientExecutionException( - "EXE0037", - ErrorLevel.WARN, - CommonErrMsg.ExecutionErr, - "Got blank ResultLocation from server. Job may not have result-set. Will not try to retrieve any Result"); - } - - resultData.updateByOperResult( - getJobOperator() - .queryResultSetPaths( - resultData.getUser(), resultData.getJobID(), resultData.getResultLocation())); - if (resultData.getResultSetPaths() == null || resultData.getResultSetPaths().length == 0) { - String msg = "Your job got no result."; - logger.warn(msg); - resultData.sendResultFin(); // inform listener to stop - resultData.setHasResult(false); - return; - } - - try { - resultData.setHasResult(true); - Thread resultRetriever = new Thread(() -> queryResultLoop(resultData), "Result-Retriever"); - SchedulerUtils.getCachedThreadPoolExecutor().execute(resultRetriever); - } catch (Exception e) { - logger.error("Failed to retrieve result", e); - throw e; - } - } - - public void queryResultLoop(LinkisResultData data) { - boolean hasNext = true; - int retryCnt = 0; - final int MAX_RETRY = 30; // continues fails for 250s, then exit - int idx = 0; - try { - while (hasNext) { - try { - hasNext = queryOneResult(data, idx); - } catch (LinkisClientRuntimeException e) { - logger.error("Cannot get result:", e); - retryCnt++; - if (retryCnt >= MAX_RETRY) { - logger.error( - "Continuously failing to query result for " - + MAX_RETRY * (MAX_RETRY + 2) * 500 / 1000 - + "s. Will no longer try to query result", - e); - return; - } else { - hasNext = true; - } - Utils.doSleepQuietly(500l + 500l * retryCnt); // maybe server problem. sleep longer - continue; - } - idx++; - } - } catch (Exception e) { - logger.error("Something goes wrong. Job Result may be incomplete", e); - throw e; - } finally { - data.sendResultFin(); - } - } - - private boolean queryOneResult(LinkisResultData data, int idxResultSet) { - Integer curPage = 1; - boolean hasNextResult = true; - boolean hasNextPage = true; - while (hasNextPage) { - data.updateByOperResult( - getJobOperator() - .queryResultSetGivenResultSetPath( - data.getResultSetPaths(), - idxResultSet, - data.getUser(), - curPage, - AppConstants.RESULTSET_PAGE_SIZE)); - if (data.hasNextResultPage() == null) { - throw new LinkisClientExecutionException( - "EXE0040", - ErrorLevel.ERROR, - CommonErrMsg.ExecutionResultErr, - "Something foes wrong. Got null as \'hasNextPage\'."); - } - hasNextPage = data.hasNextResultPage(); - - curPage++; - hasNextResult = idxResultSet + 1 < data.getResultSetPaths().length; - } - return hasNextResult; - } - - public void doKill() { - data.updateByOperResult(getJobOperator().queryJobInfo(jobDesc.getUser(), jobDesc.getJobID())); - if (data.getUser() == null || data.getJobID() == null) { - throw new LinkisClientExecutionException( - "EXE0036", ErrorLevel.ERROR, CommonErrMsg.ExecutionErr, "user or jobID is null"); - } - if (data.getJobStatus() == null) { - throw new LinkisClientExecutionException( - "EXE0038", ErrorLevel.ERROR, CommonErrMsg.ExecutionErr, "jobStatus is null"); - } - String msg; - if (data.getJobStatus().isJobCancelled()) { - msg = "Kill job aborted: Job has already been canceled."; - data.setSuccess(false); - data.setMessage(msg); - } else if (data.getJobStatus().isJobFinishedState()) { - msg = "Kill job aborted: Job is already in finished-state(SUCCEED/FAILED)."; - data.setSuccess(false); - data.setMessage(msg); - // throw new LinkisClientExecutionException(JobStatus.FAILED, "EXE0004", - // ErrorLevel.ERROR, CommonErrMsg.ExecutionErr, msg); - } else { - try { - data.updateByOperResult( - getJobOperator().queryJobInfo(jobDesc.getUser(), jobDesc.getJobID())); - data.updateByOperResult( - getJobOperator().kill(data.getUser(), data.getJobID(), data.getExecID())); - } catch (Exception e) { - data.setSuccess(false); - data.setMessage("Exception thrown when trying to send kill request"); - data.setException(e); - } - msg = "Kill request has been sent"; - LogUtils.getPlaintTextLogger().info(msg); - int retryCnt = 0; - final int MAX_RETRY = 5 * 6; - while (!data.getJobStatus().isJobFinishedState() && !data.getJobStatus().isJobCancelled()) { - CommonUtils.doSleepQuietly(CommonConstants.JOB_QUERY_SLEEP_MILLS); - try { - data.updateByOperResult( - getJobOperator().queryJobInfo(jobDesc.getUser(), jobDesc.getJobID())); - retryCnt = 0; // if exception then will not go here - } catch (Exception e) { - retryCnt++; - CommonUtils.doSleepQuietly(5 * CommonConstants.JOB_QUERY_SLEEP_MILLS); - if (retryCnt >= MAX_RETRY) { - data.setSuccess(false); - data.setMessage( - MessageFormat.format( - "After send kill. Client cannot get jobStatus from server continuously for {0} seconds. Client aborted. Assume kill failed! Error message: \n", - MAX_RETRY * 5 * CommonConstants.JOB_QUERY_SLEEP_MILLS)); - data.setException(e); - return; - } - } - } - if (data.getJobStatus().isJobFinishedState() && !data.getJobStatus().isJobCancelled()) { - msg = "Kill Failed: Job Current status: " + data.getJobStatus(); - data.setSuccess(false); - data.setMessage(msg); - // throw new LinkisClientExecutionException(JobStatus.FAILED, - // "EXE0004", ErrorLevel.ERROR, CommonErrMsg.ExecutionErr, msg); - } else if (data.getJobStatus().isJobCancelled()) { - msg = - MessageFormat.format( - "Kill successful: jobId={0}, status={1}.", data.getJobID(), data.getJobStatus()); - data.setSuccess(true); - data.setMessage(msg); - // LogUtils.getPlaintTextLogger().info(msg); - } - } - return; - } - - @Override - public TerminateToken getTerminateToken() { - return terminateToken; - } - - public void setTerminateToken(TerminateToken terminateToken) { - this.terminateToken = terminateToken; - } - - @Override - public void terminate() throws LinkisClientRuntimeException { - return; - } -} diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/interactor/job/LinkisOnceJob.java b/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/interactor/job/LinkisOnceJob.java deleted file mode 100644 index 505b08cf26..0000000000 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/interactor/job/LinkisOnceJob.java +++ /dev/null @@ -1,308 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.cli.application.interactor.job; - -import org.apache.linkis.cli.application.constants.AppConstants; -import org.apache.linkis.cli.application.interactor.job.data.LinkisJobData; -import org.apache.linkis.cli.application.interactor.job.data.LinkisLogData; -import org.apache.linkis.cli.application.interactor.job.data.LinkisOnceJobData; -import org.apache.linkis.cli.application.interactor.job.desc.LinkisOnceDesc; -import org.apache.linkis.cli.application.utils.Utils; -import org.apache.linkis.cli.common.entity.job.JobDescription; -import org.apache.linkis.cli.common.entity.operator.JobOperator; -import org.apache.linkis.cli.common.exception.LinkisClientRuntimeException; -import org.apache.linkis.cli.common.exception.error.ErrorLevel; -import org.apache.linkis.cli.core.constants.CommonConstants; -import org.apache.linkis.cli.core.exception.LinkisClientExecutionException; -import org.apache.linkis.cli.core.exception.error.CommonErrMsg; -import org.apache.linkis.cli.core.interactor.job.*; -import org.apache.linkis.cli.core.utils.CommonUtils; -import org.apache.linkis.cli.core.utils.LogUtils; -import org.apache.linkis.cli.core.utils.SchedulerUtils; - -import java.text.MessageFormat; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public class LinkisOnceJob extends LinkisJob - implements ManagableBackendJob, - LogAccessibleJob, - ResultAccessibleJob, - AsyncBackendJob, - TerminatableJob { - - private static final Logger logger = LoggerFactory.getLogger(LinkisOnceJob.class); - - private LinkisOnceDesc jobDesc; - private LinkisOnceJobData data; - private TerminateToken terminateToken = new TerminateToken(); - private Boolean isAsync = false; - - public void setAsync(Boolean async) { - isAsync = async; - } - - public Boolean isAsync() { - return isAsync; - } - - @Override - public JobDescription getJobDesc() { - return jobDesc; - } - - public void setJobDesc(LinkisOnceDesc desc) { - this.jobDesc = desc; - } - - @Override - public TerminateToken getTerminateToken() { - return terminateToken; - } - - @Override - public LinkisJobData getJobData() { - return data; - } - - public void setJobData(LinkisOnceJobData data) { - this.data = data; - } - - @Override - public JobOperator getJobOperator() { - return null; - } - - /** AsyncBackendJob */ - @Override - public void submit() throws LinkisClientRuntimeException { - StringBuilder infoBuilder = new StringBuilder(); - infoBuilder - .append("connecting to linkis gateway:") - .append(data.getOnceJobAdapter().getServerUrl()); - LogUtils.getInformationLogger().info(infoBuilder.toString()); - data.getOnceJobAdapter().submit(); - data.getOnceJobAdapter().updateStatus(); - infoBuilder.setLength(0); - infoBuilder - .append("JobId:") - .append(data.getJobID()) - .append(System.lineSeparator()) - .append("ExecId:") - .append(data.getExecID()); - LogUtils.getPlaintTextLogger().info(infoBuilder.toString()); - if (isAsync) { - data.setSuccess(data.getJobStatus() != null && data.getJobStatus().isJobSubmitted()); - } - } - - @Override - public void updateJobStatus() throws LinkisClientRuntimeException { - if (!data.getJobStatus().isJobFinishedState()) { - data.getOnceJobAdapter().updateStatus(); - String log2 = - "\n---------------------------------------------------\n" - + "\ttask " - + data.getJobID() - + " status is " - + data.getJobStatus() - + ", progress : " - + data.getJobProgress() - + "\n---------------------------------------------------"; - logger.info(log2); - } - } - - @Override - public void waitJobComplete() throws LinkisClientRuntimeException { - data.getOnceJobAdapter().waitForComplete(); - updateJobStatus(); - data.setSuccess(data.getJobStatus() != null && data.getJobStatus().isJobSuccess()); - waitIncLogComplete(data); - } - - /** TerminatableJob */ - @Override - public void terminate() throws LinkisClientRuntimeException { - terminateToken.setTerminate(); - doKill(); - } - - @Override - public void startRetrieveResult() { - // TODO:wait for OnceJob to support this feature - data.sendResultFin(); - } - - @Override - public void startRetrieveLog() { - data.setIncLogMode(true); - startRetrieveLogInternal(data); - } - - public void startRetrieveLogInternal(LinkisOnceJobData jobData) { - if (!(jobData instanceof LinkisLogData)) { - throw new LinkisClientExecutionException( - "EXE0034", ErrorLevel.ERROR, CommonErrMsg.ExecutionErr, "JobData is not LinkisLogData"); - } - if (jobData.getUser() == null || jobData.getJobID() == null) { - throw new LinkisClientExecutionException( - "EXE0036", ErrorLevel.ERROR, CommonErrMsg.ExecutionErr, "user or jobID is null"); - } - LinkisOnceJobData logData = jobData; - if (logData.getJobStatus() != null) { - try { - Thread logConsumer = new Thread(() -> logData.notifyLogListener(), "Log-Consumer"); - Thread logRetriever = new Thread(() -> queryLogLoop(logData), "Log-Retriever"); - SchedulerUtils.getCachedThreadPoolExecutor().execute(logRetriever); - SchedulerUtils.getCachedThreadPoolExecutor().execute(logConsumer); - } catch (Exception e) { - logger.warn("Failed to retrieve log", e); - } - } - } - - private void queryJobLogOneIteration(LinkisOnceJobData data) throws LinkisClientRuntimeException { - try { - data.getOnceJobAdapter().queryJobLogOneIteration(); - // - // data.updateByOperResult(getJobOperator().queryRunTimeLogFromLine(data.getUser(), - // data.getJobID(), data.getExecID(), fromLine)); - } catch (Exception e) { - // job is finished while we start query log(but request is not send). - // then probably server cache is gone and we got a exception here. - // however we cannot know if this happens based on the exception message - logger.warn( - "Caught exception when querying runtime-log. Probably server-side has close stream. Will try openLog api if Job is completed.", - e); - if (data.getJobStatus().isJobFinishedState()) { - CommonUtils.doSleepQuietly(500l); - // - // data.updateByOperResult(getJobOperator().queryPersistedLogFromLine(data.getUser(), - // data.getJobID(), data.getExecID(), fromLine)); - } - } - } - - public void queryLogLoop(LinkisOnceJobData data) { - boolean hasNext = true; - int retryCnt = 0; - final int MAX_RETRY = 12; // continues fails for 90s, then exit thread - try { - while (hasNext) { - try { - queryJobLogOneIteration(data); - } catch (Exception e) { - logger.error("Cannot get inc-log:", e); - // and yes sometimes server may not be able to prepare persisted-log - retryCnt++; - if (retryCnt >= MAX_RETRY) { - logger.error( - "Continuously failing to query inc-log for " - + MAX_RETRY * (MAX_RETRY + 2) * 500 / 1000 - + "s. Will no longer try to query log", - e); - break; - } - Utils.doSleepQuietly(500l + 500l * retryCnt); // maybe server problem. sleep longer - continue; - } - retryCnt = 0; - if (data.isIncLogMode()) { - hasNext = - data.hasNextLogLine() == null - ? !data.getJobStatus().isJobFinishedState() - : data.hasNextLogLine(); - } else { - hasNext = false; - } - if (hasNext) { - String msg = - MessageFormat.format( - "Job is still running, status={0}, progress={1}", - data.getJobStatus(), String.valueOf(data.getJobProgress() * 100) + "%"); - logger.info(msg); - } - Utils.doSleepQuietly(AppConstants.JOB_QUERY_SLEEP_MILLS); - } - } catch (Exception e) { - logger.error("Something goes wrong. Job Log may be incomplete", e); - } finally { - data.sendLogFin(); - } - } - - private void waitIncLogComplete(LinkisJobData data) { - if (!(data instanceof LinkisOnceJobData)) { - return; - } - int retry = 0; - int MAX_RETRY = 300; // wait for 10 minutes after job finish - while (retry++ < MAX_RETRY) { - if (((LinkisOnceJobData) data).logFinReceived()) { - return; - } - CommonUtils.doSleepQuietly(CommonConstants.JOB_QUERY_SLEEP_MILLS); - } - String msg = - "Job is in Finished state(SUCCEED/FAILED/CANCELED) but client keep querying inclog for " - + (MAX_RETRY * CommonConstants.JOB_QUERY_SLEEP_MILLS / 1000) - + "seconds. Execution ends forcefully. Next will try handle execution result."; - logger.warn(msg); - LogUtils.getInformationLogger().warn(msg); - } - - // /** - // * LogAccessibleJob - // */ - // @Override - // public void startRetrieveLog() { - - // } - - /** ManagableBackendJob */ - @Override - public void doManage() throws LinkisClientRuntimeException {} - - @Override - public boolean isSuccess() { - return data.isSuccess(); - } - - private void doKill() { - String msg; - if (data.getJobStatus().isJobCancelled()) { - msg = "Kill job aborted: Job is failed or has already been canceled."; - data.setSuccess(false); - data.setMessage(msg); - } else if (data.getJobStatus().isJobFinishedState()) { - msg = "Kill job aborted: Job is already in finished-state(SUCCEED/FAILED)."; - data.setSuccess(false); - data.setMessage(msg); - // throw new LinkisClientExecutionException(JobStatus.FAILED, "EXE0004", - // ErrorLevel.ERROR, CommonErrMsg.ExecutionErr, msg); - } else { - data.getOnceJobAdapter().kill(); - updateJobStatus(); - data.setSuccess(true); - data.setMessage("successfully killed job"); - } - } -} diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/interactor/job/LinkisSubmitJob.java b/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/interactor/job/LinkisSubmitJob.java deleted file mode 100644 index d1e71e4e8d..0000000000 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/interactor/job/LinkisSubmitJob.java +++ /dev/null @@ -1,282 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.cli.application.interactor.job; - -import org.apache.linkis.cli.application.interactor.job.data.LinkisJobData; -import org.apache.linkis.cli.application.interactor.job.data.LinkisLogData; -import org.apache.linkis.cli.application.interactor.job.data.LinkisResultData; -import org.apache.linkis.cli.application.interactor.job.desc.LinkisJobManDesc; -import org.apache.linkis.cli.application.interactor.job.desc.LinkisSubmitDesc; -import org.apache.linkis.cli.application.operator.ujes.LinkisJobOperator; -import org.apache.linkis.cli.common.entity.operator.JobOperator; -import org.apache.linkis.cli.common.exception.LinkisClientRuntimeException; -import org.apache.linkis.cli.common.exception.error.ErrorLevel; -import org.apache.linkis.cli.core.constants.CommonConstants; -import org.apache.linkis.cli.core.exception.LinkisClientExecutionException; -import org.apache.linkis.cli.core.exception.error.CommonErrMsg; -import org.apache.linkis.cli.core.interactor.job.*; -import org.apache.linkis.cli.core.utils.CommonUtils; -import org.apache.linkis.cli.core.utils.LogUtils; - -import org.apache.commons.lang3.StringUtils; -import org.apache.commons.lang3.exception.ExceptionUtils; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public class LinkisSubmitJob extends LinkisJob - implements AsyncBackendJob, LogAccessibleJob, ResultAccessibleJob, TerminatableJob { - private static final Logger logger = LoggerFactory.getLogger(LinkisSubmitJob.class); - - private LinkisSubmitDesc jobDesc; - private LinkisJobData data; - private TerminateToken terminateToken = new TerminateToken(); - private LinkisManageJob manageJob = new LinkisManageJob(); - private Boolean isAsync = false; - - public void setAsync(Boolean async) { - isAsync = async; - } - - public Boolean isAsync() { - return isAsync; - } - - @Override - public LinkisJobOperator getJobOperator() { - if (!(super.getJobOperator() instanceof LinkisJobOperator)) { - throw new LinkisClientExecutionException( - "EXE0003", - ErrorLevel.ERROR, - CommonErrMsg.ExecutionInitErr, - "JobOperator of LinkisManageJob should be instance of LinkisJobOperator"); - } - return (LinkisJobOperator) super.getJobOperator(); - } - - @Override - public void setOperator(JobOperator operator) { - if (!(operator instanceof LinkisJobOperator)) { - throw new LinkisClientExecutionException( - "EXE0003", - ErrorLevel.ERROR, - CommonErrMsg.ExecutionInitErr, - "JobOperator of LinkisManageJob should be instance of LinkisJobOperator"); - } - manageJob.setOperator(operator); - super.setOperator(operator); - } - - @Override - public LinkisSubmitDesc getJobDesc() { - return jobDesc; - } - - public void setJobDesc(LinkisSubmitDesc jobDesc) { - this.jobDesc = jobDesc; - } - - @Override - public LinkisJobData getJobData() { - return data; - } - - public void setJobData(LinkisJobData data) { - manageJob.setJobData(data); - this.data = data; - } - - @Override - public TerminateToken getTerminateToken() { - return terminateToken; - } - - public void setTerminateToken(TerminateToken terminateToken) { - this.terminateToken = terminateToken; - } - - @Override - public void submit() throws LinkisClientRuntimeException { - StringBuilder infoBuilder = new StringBuilder(); - infoBuilder.append("connecting to linkis gateway:").append(getJobOperator().getServerUrl()); - LogUtils.getInformationLogger().info(infoBuilder.toString()); - data.updateByOperResult(getJobOperator().submit(jobDesc)); - CommonUtils.doSleepQuietly(2000l); - LinkisJobManDesc jobManDesc = new LinkisJobManDesc(); - jobManDesc.setJobId(data.getJobID()); - jobManDesc.setUser(data.getUser()); - manageJob.setJobDesc(jobManDesc); - data.updateByOperResult(getJobOperator().queryJobInfo(data.getUser(), data.getJobID())); - infoBuilder.setLength(0); - infoBuilder - .append("JobId:") - .append(data.getJobID()) - .append(System.lineSeparator()) - .append("TaskId:") - .append(data.getJobID()) - .append(System.lineSeparator()) - .append("ExecId:") - .append(data.getExecID()); - LogUtils.getPlaintTextLogger().info(infoBuilder.toString()); - if (isAsync) { - data.setSuccess(data.getJobStatus() != null && data.getJobStatus().isJobSubmitted()); - } - } - - @Override - public void updateJobStatus() throws LinkisClientRuntimeException { - data.updateByOperResult(getJobOperator().queryJobInfo(data.getUser(), data.getJobID())); - getJobOperator().queryJobStatus(data.getUser(), data.getJobID(), data.getExecID()); - String log2 = - "\n---------------------------------------------------\n" - + "\ttask " - + data.getJobID() - + " status is " - + data.getJobStatus() - + ", progress : " - + data.getJobProgress() - + "\n---------------------------------------------------"; - logger.info(log2); - } - - @Override - public void waitJobComplete() throws LinkisClientRuntimeException { - int retryCnt = 0; - final int MAX_RETRY = 30; - while (!data.getJobStatus().isJobFinishedState()) { - // query progress - try { - data.updateByOperResult(getJobOperator().queryJobInfo(data.getUser(), data.getJobID())); - getJobOperator().queryJobStatus(data.getUser(), data.getJobID(), data.getExecID()); - } catch (Exception e) { - logger.warn("", e); - retryCnt++; - if (retryCnt >= MAX_RETRY) { - throw new LinkisClientExecutionException( - "EXE0013", - ErrorLevel.ERROR, - CommonErrMsg.ExecutionErr, - "Cannot get jobStatus from server continuously for {0} seconds. Client aborted! Error message: \n", - MAX_RETRY * 5 * CommonConstants.JOB_QUERY_SLEEP_MILLS, - e); - } - CommonUtils.doSleepQuietly( - 5 * CommonConstants.JOB_QUERY_SLEEP_MILLS); // maybe server problem. sleep - // longer - continue; - } - retryCnt = 0; // reset counter - checkJobAvailability(data); - CommonUtils.doSleepQuietly(CommonConstants.JOB_QUERY_SLEEP_MILLS); - } - data.setSuccess(data.getJobStatus() != null && data.getJobStatus().isJobSuccess()); - waitIncLogComplete(data); - } - - private void waitIncLogComplete(LinkisJobData data) { - if (!(data instanceof LinkisLogData)) { - return; - } - int retry = 0; - int MAX_RETRY = 300; // wait for 10 minutes after job finish - while (retry++ < MAX_RETRY) { - if (((LinkisLogData) data).logFinReceived()) { - return; - } - CommonUtils.doSleepQuietly(CommonConstants.JOB_QUERY_SLEEP_MILLS); - } - String msg = - "Job is in Finished state(SUCCEED/FAILED/CANCELED) but client keep querying inclog for " - + (MAX_RETRY * CommonConstants.JOB_QUERY_SLEEP_MILLS / 1000) - + "seconds. Execution ends forcefully. Next will try handle execution result."; - logger.warn(msg); - LogUtils.getInformationLogger().warn(msg); - } - - @Override - public void startRetrieveResult() { - try { - manageJob.startRetrieveResult(); - data.setSuccess(true); - } catch (LinkisClientExecutionException e) { - if (e.getCode().equals("EXE0037")) { - data.setSuccess(true); - LogUtils.getInformationLogger().warn(e.getMessage()); - } else { - data.setSuccess(false); - data.setException(e); - } - ((LinkisResultData) data).sendResultFin(); // inform listener to stop - } catch (Exception e) { - data.setSuccess(false); - data.setException(e); - ((LinkisResultData) data).sendResultFin(); // inform listener to stop - } - } - - @Override - public void startRetrieveLog() { - if (!(data instanceof LinkisLogData)) { - throw new LinkisClientExecutionException( - "EXE0034", ErrorLevel.ERROR, CommonErrMsg.ExecutionErr, "JobData is not LinkisLogData"); - } - LinkisLogData dataCopy; - try { - dataCopy = ((LinkisLogData) data).clone(); // make a copy to avoid race condition - } catch (CloneNotSupportedException e) { - throw new LinkisClientExecutionException( - "EXE0035", ErrorLevel.ERROR, CommonErrMsg.ExecutionErr, "logData is not Cloneable", e); - } - dataCopy.setIncLogMode(true); - manageJob.startRetrieveLogInternal(dataCopy); - } - - @Override - public void terminate() throws LinkisClientRuntimeException { - terminateToken.setTerminate(); - // kill job if job is submitted - if (StringUtils.isNotBlank(data.getJobID())) { - System.out.println("\nKilling job: " + data.getJobID()); - try { - manageJob.doKill(); - if (data.getJobStatus().isJobCancelled()) { - System.out.println("Successfully killed job: " + data.getJobID() + " on exit"); - } else { - System.out.println( - "Failed to kill job: " - + data.getJobID() - + " on exit. Current job status: " - + data.getJobStatus()); - } - } catch (Exception e) { - System.out.println("Failed to kill job: " + data.getJobID() + " on exit"); - System.out.println(ExceptionUtils.getStackTrace(e)); - } - } - } - - private void checkJobAvailability(LinkisJobData data) throws LinkisClientRuntimeException { - if (data.getJobStatus().isJobAbnormalStatus()) { - throw new LinkisClientExecutionException( - "EXE0006", - ErrorLevel.ERROR, - CommonErrMsg.ExecutionErr, - "Job is in abnormal status: " + CommonUtils.GSON.toJson(data)); - } - } -} diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/interactor/job/builder/LinkisManageJobBuilder.java b/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/interactor/job/builder/LinkisManageJobBuilder.java deleted file mode 100644 index ef5648b993..0000000000 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/interactor/job/builder/LinkisManageJobBuilder.java +++ /dev/null @@ -1,140 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.cli.application.interactor.job.builder; - -import org.apache.linkis.cli.application.constants.AppKeys; -import org.apache.linkis.cli.application.interactor.job.LinkisManageJob; -import org.apache.linkis.cli.application.interactor.job.data.LinkisJobData; -import org.apache.linkis.cli.application.interactor.job.data.LinkisJobDataImpl; -import org.apache.linkis.cli.application.interactor.job.desc.LinkisJobManDesc; -import org.apache.linkis.cli.application.observer.listener.LinkisClientListener; -import org.apache.linkis.cli.application.operator.ujes.LinkisJobOperator; -import org.apache.linkis.cli.application.utils.ExecutionUtils; -import org.apache.linkis.cli.common.entity.present.PresentWay; -import org.apache.linkis.cli.common.exception.LinkisClientRuntimeException; -import org.apache.linkis.cli.common.exception.error.ErrorLevel; -import org.apache.linkis.cli.core.exception.error.CommonErrMsg; -import org.apache.linkis.cli.core.interactor.job.JobBuilder; -import org.apache.linkis.cli.core.operator.JobOperatorFactory; -import org.apache.linkis.cli.core.present.PresentModeImpl; -import org.apache.linkis.cli.core.present.PresentWayImpl; - -import org.apache.commons.lang3.StringUtils; - -import java.util.HashSet; -import java.util.Set; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public class LinkisManageJobBuilder extends JobBuilder { - private static Logger logger = LoggerFactory.getLogger(LinkisSubmitJobBuilder.class); - - LinkisClientListener logListener; - - public LinkisManageJobBuilder setLogListener(LinkisClientListener observer) { - this.logListener = observer; - return this; - } - - @Override - protected LinkisJobManDesc buildJobDesc() { - LinkisJobManDesc desc = new LinkisJobManDesc(); - String osUser = sysVarAccess.getVar(String.class, AppKeys.LINUX_USER_KEY); - String[] adminUsers = StringUtils.split(AppKeys.ADMIN_USERS, ','); - Set adminSet = new HashSet<>(); - for (String admin : adminUsers) { - adminSet.add(admin); - } - String submitUsr = ExecutionUtils.getSubmitUser(stdVarAccess, osUser, adminSet); - - String jobId = null; - if (stdVarAccess.hasVar(AppKeys.LINKIS_CLIENT_KILL_OPT)) { - jobId = stdVarAccess.getVar(String.class, AppKeys.LINKIS_CLIENT_KILL_OPT); - } else if (stdVarAccess.hasVar(AppKeys.LINKIS_CLIENT_STATUS_OPT)) { - jobId = stdVarAccess.getVar(String.class, AppKeys.LINKIS_CLIENT_STATUS_OPT); - } else if (stdVarAccess.hasVar(AppKeys.LINKIS_CLIENT_DESC_OPT)) { - jobId = stdVarAccess.getVar(String.class, AppKeys.LINKIS_CLIENT_DESC_OPT); - } else if (stdVarAccess.hasVar(AppKeys.LINKIS_CLIENT_LOG_OPT)) { - jobId = stdVarAccess.getVar(String.class, AppKeys.LINKIS_CLIENT_LOG_OPT); - } else if (stdVarAccess.hasVar(AppKeys.LINKIS_CLIENT_RESULT_OPT)) { - jobId = stdVarAccess.getVar(String.class, AppKeys.LINKIS_CLIENT_RESULT_OPT); - } else if (stdVarAccess.hasVar(AppKeys.LINKIS_CLIENT_LIST_OPT)) { - jobId = stdVarAccess.getVar(String.class, AppKeys.LINKIS_CLIENT_LIST_OPT); - } - - desc.setJobId(jobId); - desc.setUser(submitUsr); - return desc; - } - - @Override - protected LinkisJobData buildJobData() { - LinkisJobDataImpl data = new LinkisJobDataImpl(); - if (logListener == null) { - logger.warn("logListener is not registered, will not be able to display log"); - } else { - data.registerincLogListener(logListener); - } - return data; - } - - @Override - protected LinkisJobOperator buildJobOperator() { - LinkisJobOperator oper; - try { - oper = (LinkisJobOperator) JobOperatorFactory.getReusable(AppKeys.REUSABLE_UJES_CLIENT); - } catch (Exception e) { - throw new LinkisClientRuntimeException( - "BLD0012", - ErrorLevel.ERROR, - CommonErrMsg.BuilderBuildErr, - "Failed to get a valid operator.", - e); - } - return oper; - } - - @Override - protected PresentWay buildPresentWay() { - PresentWayImpl presentWay = new PresentWayImpl(); - String outputPath = stdVarAccess.getVar(String.class, AppKeys.LINKIS_CLIENT_COMMON_OUTPUT_PATH); - - presentWay.setPath(outputPath); - presentWay.setMode(PresentModeImpl.STDOUT); - if (StringUtils.isNotBlank(outputPath)) { - presentWay.setMode(PresentModeImpl.TEXT_FILE); - } - - return presentWay; - } - - @Override - protected LinkisManageJob getTargetNewInstance() { - return new LinkisManageJob(); - } - - @Override - public LinkisManageJob build() { - ((LinkisManageJob) targetObj).setJobDesc(buildJobDesc()); - ((LinkisManageJob) targetObj).setJobData(buildJobData()); - targetObj.setOperator(buildJobOperator()); - targetObj.setPresentWay(buildPresentWay()); - return (LinkisManageJob) super.build(); - } -} diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/interactor/job/builder/LinkisOnceJobBuilder.java b/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/interactor/job/builder/LinkisOnceJobBuilder.java deleted file mode 100644 index 15a2eb3ac3..0000000000 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/interactor/job/builder/LinkisOnceJobBuilder.java +++ /dev/null @@ -1,239 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.cli.application.interactor.job.builder; - -import org.apache.linkis.cli.application.constants.AppConstants; -import org.apache.linkis.cli.application.constants.AppKeys; -import org.apache.linkis.cli.application.constants.LinkisKeys; -import org.apache.linkis.cli.application.interactor.job.LinkisOnceJob; -import org.apache.linkis.cli.application.interactor.job.data.LinkisOnceJobData; -import org.apache.linkis.cli.application.interactor.job.data.SimpleOnceJobAdapter; -import org.apache.linkis.cli.application.interactor.job.desc.LinkisOnceDesc; -import org.apache.linkis.cli.application.observer.listener.LinkisClientListener; -import org.apache.linkis.cli.application.utils.ExecutionUtils; -import org.apache.linkis.cli.common.entity.operator.JobOperator; -import org.apache.linkis.cli.common.entity.present.PresentWay; -import org.apache.linkis.cli.core.interactor.job.JobBuilder; -import org.apache.linkis.cli.core.present.PresentModeImpl; -import org.apache.linkis.cli.core.present.PresentWayImpl; - -import org.apache.commons.lang3.StringUtils; - -import java.util.*; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public class LinkisOnceJobBuilder extends JobBuilder { - private static Logger logger = LoggerFactory.getLogger(LinkisSubmitJobBuilder.class); - - private LinkisClientListener logListener; - private Boolean isAsync = false; - private SimpleOnceJobAdapter onceJobAdapter = new SimpleOnceJobAdapter(); - - public LinkisOnceJobBuilder setLogListener(LinkisClientListener observer) { - this.logListener = observer; - return this; - } - - public LinkisOnceJobBuilder setAsync(Boolean async) { - isAsync = async; - return this; - } - - @Override - protected LinkisOnceJob getTargetNewInstance() { - return new LinkisOnceJob(); - } - - @Override - protected LinkisOnceDesc buildJobDesc() { - LinkisOnceDesc desc = new LinkisOnceDesc(); - - desc.setStdVarAccess(stdVarAccess); - desc.setSysVarAccess(sysVarAccess); - - Map confMap = stdVarAccess.getVar(Map.class, AppKeys.JOB_PARAM_CONF); - Map runtimeMap = stdVarAccess.getVar(Map.class, AppKeys.JOB_PARAM_RUNTIME); - Map varMap = stdVarAccess.getVar(Map.class, AppKeys.JOB_PARAM_VAR); - Map labelMap = stdVarAccess.getVar(Map.class, AppKeys.JOB_LABEL); - Map sourceMap = stdVarAccess.getVar(Map.class, AppKeys.JOB_SOURCE); - Map executionMap = stdVarAccess.getVar(Map.class, AppKeys.JOB_EXEC); - Map jobContentMap = stdVarAccess.getVar(Map.class, AppKeys.JOB_CONTENT); - - confMap = confMap == null ? new HashMap<>() : confMap; - runtimeMap = runtimeMap == null ? new HashMap<>() : runtimeMap; - varMap = varMap == null ? new HashMap<>() : varMap; - labelMap = labelMap == null ? new HashMap<>() : labelMap; - sourceMap = sourceMap == null ? new HashMap<>() : sourceMap; - executionMap = executionMap == null ? new HashMap<>() : executionMap; - jobContentMap = jobContentMap == null ? new HashMap<>() : jobContentMap; - - confMap = ProcessKeyUtils.removePrefixForKeysInMap(confMap); - runtimeMap = ProcessKeyUtils.removePrefixForKeysInMap(runtimeMap); - labelMap = ProcessKeyUtils.removePrefixForKeysInMap(labelMap); - sourceMap = ProcessKeyUtils.removePrefixForKeysInMap(sourceMap); - executionMap = ProcessKeyUtils.removePrefixForKeysInMap(executionMap); - jobContentMap = ProcessKeyUtils.removePrefixForKeysInMap(jobContentMap); - - for (String key : stdVarAccess.getAllVarKeys()) { - Object val = stdVarAccess.getVar(Object.class, key); - if (!(val instanceof Map) && val != null) { - // note that we allow it to overwrite existing values in map - if (StringUtils.startsWithIgnoreCase(key, AppKeys.JOB_PARAM_CONF)) { - ProcessKeyUtils.removePrefixAndPutValToMap(confMap, key, val, AppKeys.JOB_PARAM_CONF); - } else if (StringUtils.startsWithIgnoreCase(key, AppKeys.JOB_PARAM_VAR)) { - ProcessKeyUtils.removePrefixAndPutValToMap(varMap, key, val, AppKeys.JOB_PARAM_VAR); - } else if (StringUtils.startsWithIgnoreCase(key, AppKeys.JOB_PARAM_RUNTIME)) { - ProcessKeyUtils.removePrefixAndPutValToMap( - runtimeMap, key, val, AppKeys.JOB_PARAM_RUNTIME); - } else if (StringUtils.startsWithIgnoreCase(key, AppKeys.JOB_EXEC)) { - ProcessKeyUtils.removePrefixAndPutValToMap(executionMap, key, val, AppKeys.JOB_EXEC); - } else if (StringUtils.startsWithIgnoreCase(key, AppKeys.JOB_LABEL)) { - ProcessKeyUtils.removePrefixAndPutValToMap(labelMap, key, val, AppKeys.JOB_LABEL); - } else if (StringUtils.startsWithIgnoreCase(key, AppKeys.JOB_SOURCE)) { - ProcessKeyUtils.removePrefixAndPutValToMap(sourceMap, key, val, AppKeys.JOB_SOURCE); - } else if (StringUtils.startsWithIgnoreCase(key, AppKeys.JOB_CONTENT)) { - ProcessKeyUtils.removePrefixAndPutValToMap(jobContentMap, key, val, AppKeys.JOB_CONTENT); - } else if (StringUtils.startsWithIgnoreCase(key, AppKeys.LINKIS_CLIENT_COMMON)) { - // do nothing - } else { - // confMap.put(key, stdVarAccess.getVar(Object.class, key)); - } - } - } - - String creator; - if (!isAsync) { - creator = - stdVarAccess.getVarOrDefault( - String.class, AppKeys.JOB_COMMON_CREATOR, AppConstants.JOB_CREATOR_DEFAULT); - } else { - creator = - stdVarAccess.getVarOrDefault( - String.class, AppKeys.JOB_COMMON_CREATOR, AppConstants.JOB_CREATOR_ASYNC_DEFAULT); - } - String code = stdVarAccess.getVar(String.class, AppKeys.JOB_EXEC_CODE); - String engineType = stdVarAccess.getVar(String.class, AppKeys.JOB_LABEL_ENGINE_TYPE); - String runType = stdVarAccess.getVar(String.class, AppKeys.JOB_LABEL_CODE_TYPE); - String scriptPath = - stdVarAccess.getVarOrDefault(String.class, AppKeys.JOB_SOURCE_SCRIPT_PATH, "LinkisCli"); - - String osUser = sysVarAccess.getVar(String.class, AppKeys.LINUX_USER_KEY); - String[] adminUsers = StringUtils.split(AppKeys.ADMIN_USERS, ','); - Set adminSet = new HashSet<>(); - for (String admin : adminUsers) { - adminSet.add(admin); - } - String submitUsr = ExecutionUtils.getSubmitUser(stdVarAccess, osUser, adminSet); - String proxyUsr = ExecutionUtils.getProxyUser(stdVarAccess, submitUsr, adminSet); - - String enableExecuteOnce = - stdVarAccess.getVarOrDefault(String.class, AppKeys.JOB_LABEL_EXECUTEONCE, "true"); - // default executeOnce-mode - if (Boolean.parseBoolean(enableExecuteOnce)) { - labelMap.put(LinkisKeys.KEY_EXECUTEONCE, ""); - } else { - labelMap.remove(LinkisKeys.KEY_EXECUTEONCE); - } - String codePath = stdVarAccess.getVar(String.class, AppKeys.JOB_COMMON_CODE_PATH); - Object extraArgsObj = stdVarAccess.getVar(Object.class, AppKeys.JOB_EXTRA_ARGUMENTS); - if (extraArgsObj != null - && extraArgsObj instanceof String[] - && StringUtils.isBlank(code) - && StringUtils.isBlank(codePath)) { - String[] extraArgs = (String[]) extraArgsObj; - codePath = extraArgs[0]; - if (extraArgs.length > 1) { - runtimeMap.put( - LinkisKeys.EXTRA_ARGUMENTS, Arrays.copyOfRange(extraArgs, 1, extraArgs.length)); - } - } - - if (StringUtils.isBlank(code) && StringUtils.isNotBlank(codePath)) { - code = ExecutionUtils.readFile(codePath); - } - - executionMap.put(LinkisKeys.KEY_CODE, code); - labelMap.put(LinkisKeys.KEY_ENGINETYPE, engineType); - labelMap.put(LinkisKeys.KEY_CODETYPE, runType); - labelMap.put(LinkisKeys.KEY_USER_CREATOR, proxyUsr + "-" + creator); - sourceMap.put(LinkisKeys.KEY_SCRIPT_PATH, scriptPath); - runtimeMap.put(LinkisKeys.KEY_HIVE_RESULT_DISPLAY_TBALE, true); - - desc.setCreator(creator); - desc.setParamConfMap(confMap); - desc.setParamRunTimeMap(runtimeMap); - desc.setParamVarsMap(varMap); - desc.setLabelMap(labelMap); - desc.setSourceMap(sourceMap); - desc.setExecutionMap(executionMap); - desc.setSubmitUser(submitUsr); - desc.setProxyUser(proxyUsr); - desc.setJobContentMap(jobContentMap); - - return desc; - } - - @Override - protected LinkisOnceJobData buildJobData() { - LinkisOnceJobData data = new LinkisOnceJobData(); - data.setOnceJobAdapter(this.onceJobAdapter); - if (logListener == null) { - logger.warn("logListener is not registered, will not be able to display log"); - } else { - data.registerincLogListener(logListener); - } - return data; - } - - @Override - protected JobOperator buildJobOperator() { - // OnceJob is Stateful, should not have an operator - return null; - } - - @Override - protected PresentWay buildPresentWay() { - PresentWayImpl presentWay = new PresentWayImpl(); - String outputPath = stdVarAccess.getVar(String.class, AppKeys.LINKIS_CLIENT_COMMON_OUTPUT_PATH); - - presentWay.setPath(outputPath); - presentWay.setMode(PresentModeImpl.STDOUT); - presentWay.setDisplayMetaAndLogo( - stdVarAccess.getVarOrDefault(Boolean.class, AppKeys.LINKIS_COMMON_DIAPLAY_META_LOGO, true)); - if (StringUtils.isNotBlank(outputPath)) { - presentWay.setMode(PresentModeImpl.TEXT_FILE); - } - - return presentWay; - } - - @Override - public LinkisOnceJob build() { - LinkisOnceDesc desc = buildJobDesc(); - ((LinkisOnceJob) targetObj).setJobDesc(desc); - LinkisOnceJobData data = buildJobData(); - ((LinkisOnceJob) targetObj).setJobData(data); - data.getOnceJobAdapter().init(desc); - ((LinkisOnceJob) targetObj).setAsync(isAsync); - targetObj.setOperator(buildJobOperator()); - targetObj.setPresentWay(buildPresentWay()); - return (LinkisOnceJob) super.build(); - } -} diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/interactor/job/builder/LinkisSubmitJobBuilder.java b/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/interactor/job/builder/LinkisSubmitJobBuilder.java deleted file mode 100644 index 6804f09056..0000000000 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/interactor/job/builder/LinkisSubmitJobBuilder.java +++ /dev/null @@ -1,241 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.cli.application.interactor.job.builder; - -import org.apache.linkis.cli.application.constants.AppConstants; -import org.apache.linkis.cli.application.constants.AppKeys; -import org.apache.linkis.cli.application.constants.LinkisKeys; -import org.apache.linkis.cli.application.interactor.job.LinkisSubmitJob; -import org.apache.linkis.cli.application.interactor.job.data.LinkisJobData; -import org.apache.linkis.cli.application.interactor.job.data.LinkisJobDataImpl; -import org.apache.linkis.cli.application.interactor.job.desc.LinkisSubmitDesc; -import org.apache.linkis.cli.application.observer.listener.LinkisClientListener; -import org.apache.linkis.cli.application.operator.ujes.LinkisJobOperator; -import org.apache.linkis.cli.application.utils.ExecutionUtils; -import org.apache.linkis.cli.common.entity.present.PresentWay; -import org.apache.linkis.cli.common.exception.LinkisClientRuntimeException; -import org.apache.linkis.cli.common.exception.error.ErrorLevel; -import org.apache.linkis.cli.core.exception.error.CommonErrMsg; -import org.apache.linkis.cli.core.interactor.job.JobBuilder; -import org.apache.linkis.cli.core.operator.JobOperatorFactory; -import org.apache.linkis.cli.core.present.PresentModeImpl; -import org.apache.linkis.cli.core.present.PresentWayImpl; - -import org.apache.commons.lang3.StringUtils; - -import java.util.*; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public class LinkisSubmitJobBuilder extends JobBuilder { - private static Logger logger = LoggerFactory.getLogger(LinkisSubmitJobBuilder.class); - - private LinkisClientListener logListener; - private Boolean isAsync = false; - - public LinkisSubmitJobBuilder setLogListener(LinkisClientListener observer) { - this.logListener = observer; - return this; - } - - public LinkisSubmitJobBuilder setAsync(Boolean async) { - isAsync = async; - return this; - } - - @Override - protected LinkisSubmitJob getTargetNewInstance() { - return new LinkisSubmitJob(); - } - - @Override - protected LinkisSubmitDesc buildJobDesc() { - LinkisSubmitDesc desc = new LinkisSubmitDesc(); - - Map confMap = stdVarAccess.getVar(Map.class, AppKeys.JOB_PARAM_CONF); - Map runtimeMap = stdVarAccess.getVar(Map.class, AppKeys.JOB_PARAM_RUNTIME); - Map varMap = stdVarAccess.getVar(Map.class, AppKeys.JOB_PARAM_VAR); - Map labelMap = stdVarAccess.getVar(Map.class, AppKeys.JOB_LABEL); - Map sourceMap = stdVarAccess.getVar(Map.class, AppKeys.JOB_SOURCE); - Map executionMap = stdVarAccess.getVar(Map.class, AppKeys.JOB_EXEC); - - confMap = confMap == null ? new HashMap<>() : confMap; - runtimeMap = runtimeMap == null ? new HashMap<>() : runtimeMap; - varMap = varMap == null ? new HashMap<>() : varMap; - labelMap = labelMap == null ? new HashMap<>() : labelMap; - sourceMap = sourceMap == null ? new HashMap<>() : sourceMap; - executionMap = executionMap == null ? new HashMap<>() : executionMap; - - /** remove key prefix of all keys in map type params. e.g. kv in confMap, labelMap etc. */ - confMap = ProcessKeyUtils.removePrefixForKeysInMap(confMap); - runtimeMap = ProcessKeyUtils.removePrefixForKeysInMap(runtimeMap); - labelMap = ProcessKeyUtils.removePrefixForKeysInMap(labelMap); - sourceMap = ProcessKeyUtils.removePrefixForKeysInMap(sourceMap); - executionMap = ProcessKeyUtils.removePrefixForKeysInMap(executionMap); - - /** remove key prefix of non-map type params */ - for (String key : stdVarAccess.getAllVarKeys()) { - Object val = stdVarAccess.getVar(Object.class, key); - if (!(val instanceof Map) && val != null) { - // note that we allow it to overwrite existing values in map - if (StringUtils.startsWithIgnoreCase(key, AppKeys.JOB_PARAM_CONF)) { - ProcessKeyUtils.removePrefixAndPutValToMap(confMap, key, val, AppKeys.JOB_PARAM_CONF); - } else if (StringUtils.startsWithIgnoreCase(key, AppKeys.JOB_PARAM_VAR)) { - ProcessKeyUtils.removePrefixAndPutValToMap(varMap, key, val, AppKeys.JOB_PARAM_VAR); - } else if (StringUtils.startsWithIgnoreCase(key, AppKeys.JOB_PARAM_RUNTIME)) { - ProcessKeyUtils.removePrefixAndPutValToMap( - runtimeMap, key, val, AppKeys.JOB_PARAM_RUNTIME); - } else if (StringUtils.startsWithIgnoreCase(key, AppKeys.JOB_EXEC)) { - ProcessKeyUtils.removePrefixAndPutValToMap(executionMap, key, val, AppKeys.JOB_EXEC); - } else if (StringUtils.startsWithIgnoreCase(key, AppKeys.JOB_LABEL)) { - ProcessKeyUtils.removePrefixAndPutValToMap(labelMap, key, val, AppKeys.JOB_LABEL); - } else if (StringUtils.startsWithIgnoreCase(key, AppKeys.JOB_SOURCE)) { - ProcessKeyUtils.removePrefixAndPutValToMap(sourceMap, key, val, AppKeys.JOB_SOURCE); - } else if (StringUtils.startsWithIgnoreCase(key, AppKeys.LINKIS_CLIENT_COMMON)) { - // do nothing - } else { - // confMap.put(key, stdVarAccess.getVar(Object.class, key)); - } - } - } - - String creator; - if (!isAsync) { - creator = - stdVarAccess.getVarOrDefault( - String.class, AppKeys.JOB_COMMON_CREATOR, AppConstants.JOB_CREATOR_DEFAULT); - } else { - creator = - stdVarAccess.getVarOrDefault( - String.class, AppKeys.JOB_COMMON_CREATOR, AppConstants.JOB_CREATOR_ASYNC_DEFAULT); - } - String code = stdVarAccess.getVar(String.class, AppKeys.JOB_EXEC_CODE); - String engineType = stdVarAccess.getVar(String.class, AppKeys.JOB_LABEL_ENGINE_TYPE); - String runType = stdVarAccess.getVar(String.class, AppKeys.JOB_LABEL_CODE_TYPE); - String scriptPath = - stdVarAccess.getVarOrDefault(String.class, AppKeys.JOB_SOURCE_SCRIPT_PATH, "LinkisCli"); - - String osUser = sysVarAccess.getVar(String.class, AppKeys.LINUX_USER_KEY); - String[] adminUsers = StringUtils.split(AppKeys.ADMIN_USERS, ','); - Set adminSet = new HashSet<>(); - for (String admin : adminUsers) { - adminSet.add(admin); - } - String submitUsr = ExecutionUtils.getSubmitUser(stdVarAccess, osUser, adminSet); - String proxyUsr = ExecutionUtils.getProxyUser(stdVarAccess, submitUsr, adminSet); - - String enableExecuteOnce = - stdVarAccess.getVarOrDefault(String.class, AppKeys.JOB_LABEL_EXECUTEONCE, "true"); - // default executeOnce-mode - if (Boolean.parseBoolean(enableExecuteOnce)) { - labelMap.put(LinkisKeys.KEY_EXECUTEONCE, ""); - } else { - labelMap.remove(LinkisKeys.KEY_EXECUTEONCE); - } - String codePath = stdVarAccess.getVar(String.class, AppKeys.JOB_COMMON_CODE_PATH); - Object extraArgsObj = stdVarAccess.getVar(Object.class, AppKeys.JOB_EXTRA_ARGUMENTS); - if (extraArgsObj != null - && extraArgsObj instanceof String[] - && StringUtils.isBlank(code) - && StringUtils.isBlank(codePath)) { - String[] extraArgs = (String[]) extraArgsObj; - codePath = extraArgs[0]; - if (extraArgs.length > 1) { - runtimeMap.put( - LinkisKeys.EXTRA_ARGUMENTS, Arrays.copyOfRange(extraArgs, 1, extraArgs.length)); - } - } - - if (StringUtils.isBlank(code) && StringUtils.isNotBlank(codePath)) { - code = ExecutionUtils.readFile(codePath); - } - - executionMap.put(LinkisKeys.KEY_CODE, code); - labelMap.put(LinkisKeys.KEY_ENGINETYPE, engineType); - labelMap.put(LinkisKeys.KEY_CODETYPE, runType); - labelMap.put(LinkisKeys.KEY_USER_CREATOR, proxyUsr + "-" + creator); - sourceMap.put(LinkisKeys.KEY_SCRIPT_PATH, scriptPath); - runtimeMap.put(LinkisKeys.KEY_HIVE_RESULT_DISPLAY_TBALE, true); - - desc.setCreator(creator); - desc.setParamConfMap(confMap); - desc.setParamRunTimeMap(runtimeMap); - desc.setParamVarsMap(varMap); - desc.setLabelMap(labelMap); - desc.setSourceMap(sourceMap); - desc.setExecutionMap(executionMap); - desc.setSubmitUser(submitUsr); - desc.setProxyUser(proxyUsr); - - return desc; - } - - @Override - protected LinkisJobData buildJobData() { - LinkisJobDataImpl data = new LinkisJobDataImpl(); - if (logListener == null) { - logger.warn("logListener is not registered, will not be able to display log"); - } else { - data.registerincLogListener(logListener); - } - return data; - } - - @Override - protected LinkisJobOperator buildJobOperator() { - LinkisJobOperator oper; - try { - oper = (LinkisJobOperator) JobOperatorFactory.getReusable(AppKeys.REUSABLE_UJES_CLIENT); - } catch (Exception e) { - throw new LinkisClientRuntimeException( - "BLD0012", - ErrorLevel.ERROR, - CommonErrMsg.BuilderBuildErr, - "Failed to get a valid operator.", - e); - } - return oper; - } - - @Override - protected PresentWay buildPresentWay() { - PresentWayImpl presentWay = new PresentWayImpl(); - String outputPath = stdVarAccess.getVar(String.class, AppKeys.LINKIS_CLIENT_COMMON_OUTPUT_PATH); - - presentWay.setPath(outputPath); - presentWay.setMode(PresentModeImpl.STDOUT); - presentWay.setDisplayMetaAndLogo( - stdVarAccess.getVarOrDefault(Boolean.class, AppKeys.LINKIS_COMMON_DIAPLAY_META_LOGO, true)); - if (StringUtils.isNotBlank(outputPath)) { - presentWay.setMode(PresentModeImpl.TEXT_FILE); - } - - return presentWay; - } - - @Override - public LinkisSubmitJob build() { - ((LinkisSubmitJob) targetObj).setJobDesc(buildJobDesc()); - ((LinkisSubmitJob) targetObj).setJobData(buildJobData()); - ((LinkisSubmitJob) targetObj).setAsync(isAsync); - targetObj.setOperator(buildJobOperator()); - targetObj.setPresentWay(buildPresentWay()); - return (LinkisSubmitJob) super.build(); - } -} diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/interactor/job/data/LinkisJobData.java b/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/interactor/job/data/LinkisJobData.java deleted file mode 100644 index 3d24570bf2..0000000000 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/interactor/job/data/LinkisJobData.java +++ /dev/null @@ -1,40 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.cli.application.interactor.job.data; - -import org.apache.linkis.cli.application.operator.ujes.LinkisOperResultAdapter; -import org.apache.linkis.cli.common.entity.job.JobData; - -public interface LinkisJobData extends JobData, Cloneable { - - String getExecID(); - - float getJobProgress(); - - Integer getErrCode(); - - String getErrDesc(); - - boolean isSuccess(); - - void setSuccess(boolean success); - - void updateByOperResult(LinkisOperResultAdapter adapter); - - LinkisJobData clone() throws CloneNotSupportedException; -} diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/interactor/job/data/LinkisJobDataImpl.java b/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/interactor/job/data/LinkisJobDataImpl.java deleted file mode 100644 index 227e8c2170..0000000000 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/interactor/job/data/LinkisJobDataImpl.java +++ /dev/null @@ -1,514 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.cli.application.interactor.job.data; - -import org.apache.linkis.cli.application.observer.event.LinkisClientEvent; -import org.apache.linkis.cli.application.observer.event.LogStartEvent; -import org.apache.linkis.cli.application.observer.event.TriggerEvent; -import org.apache.linkis.cli.application.observer.listener.LinkisClientListener; -import org.apache.linkis.cli.application.observer.listener.TriggerEventListener; -import org.apache.linkis.cli.application.operator.ujes.LinkisOperResultAdapter; -import org.apache.linkis.cli.common.entity.job.JobStatus; - -import java.util.Arrays; -import java.util.Date; -import java.util.LinkedList; -import java.util.List; -import java.util.concurrent.LinkedBlockingDeque; - -public class LinkisJobDataImpl - implements LinkisJobData, LinkisLogData, LinkisResultData, Cloneable { - - private String jobID; - private String user; - private JobStatus jobStatus = null; - private String message; - private Exception exception; - private String execID; - private float progress = 0.0f; - private Boolean incLogMode; - private LinkedBlockingDeque logBuffer = new LinkedBlockingDeque(); - private String logPath; // remote path for job log - private Integer nextLogLineIdx; - private Boolean hasNextLogLine; - private String resultLocation; - private String[] resultSetPaths = null; // remote paths for job result set - private LinkedBlockingDeque resultContent = new LinkedBlockingDeque<>(); - private Boolean hasNextResultPage; - private Integer errCode = null; - private String errDesc = null; - private boolean success = false; - private String instance; - private String umUser; - private String simpleExecId; - private String executionCode; - private String engineType; - private String runType; - private Long costTime; - private Date createdTime; - private Date updatedTime; - private Date engineStartTime; - private String executeApplicationName; - private String requestApplicationName; - - private LinkisClientEvent logstartEvent = new LogStartEvent(); - private TriggerEvent logFinevent = new TriggerEvent(); - private TriggerEventListener logFinListener = new TriggerEventListener(); - private TriggerEvent resultFinEvent = new TriggerEvent(); - private TriggerEventListener resultFinListener = new TriggerEventListener(); - - private boolean hasResult = true; - - { - logFinevent.register(logFinListener); - resultFinEvent.register(resultFinListener); - } - - @Override - public String getJobID() { - return jobID; - } - - public void setJobId(String jobId) { - this.jobID = jobId; - } - - @Override - public String getUser() { - return user; - } - - public void setUser(String user) { - this.user = user; - } - - @Override - public JobStatus getJobStatus() { - return jobStatus; - } - - public void setJobStatus(JobStatus jobStatus) { - this.jobStatus = jobStatus; - } - - @Override - public String getMessage() { - return message; - } - - @Override - public void setMessage(String message) { - this.message = message; - } - - @Override - public Exception getException() { - return exception; - } - - @Override - public void setException(Exception exception) { - this.exception = exception; - } - - @Override - public final String getExecID() { - return execID; - } - - public final void setExecID(String execID) { - this.execID = execID; - } - - @Override - public final float getJobProgress() { - return progress; - } - - public final void setJobProgress(float progress) { - this.progress = progress; - } - - @Override - public final String getLogPath() { - return logPath; - } - - public final void setLogPath(String logPath) { - this.logPath = logPath; - } - - @Override - public final String getResultLocation() { - return resultLocation; - } - - public final void setResultLocation(String resultLocation) { - this.resultLocation = resultLocation; - } - - @Override - public String[] getResultSetPaths() { - return resultSetPaths; - } - - public final void setResultSetPaths(String[] resultSetPaths) { - this.resultSetPaths = resultSetPaths; - } - - @Override - public Integer getErrCode() { - return errCode; - } - - public void setErrCode(Integer errCode) { - this.errCode = errCode; - } - - @Override - public String getErrDesc() { - return errDesc; - } - - public void setErrDesc(String errDesc) { - this.errDesc = errDesc; - } - - @Override - public void registerincLogListener(LinkisClientListener observer) { - this.logstartEvent.register(observer); - } - - @Override - public void notifyLogListener() { - if (this.logstartEvent.isRegistered()) { - logstartEvent.notifyObserver(logstartEvent, this); - } - } - - @Override - public boolean isIncLogMode() { - return this.incLogMode; - } - - @Override - public void setIncLogMode(boolean incLogMode) { - this.incLogMode = incLogMode; - } - - @Override - public String consumeLog() { - List logs = new LinkedList<>(); - this.logBuffer.drainTo(logs, this.logBuffer.size()); - StringBuilder tmp = new StringBuilder(); - for (String str : logs) { - tmp.append(str); - } - return tmp.toString(); - } - - public void appendLog(String log) { - this.logBuffer.add(log); - } - - @Override - public Integer getNextLogLineIdx() { - return nextLogLineIdx; - } - - public void setNextLogLineIdx(Integer nextLogLineIdx) { - this.nextLogLineIdx = nextLogLineIdx; - } - - @Override - public Boolean hasNextLogLine() { - return hasNextLogLine; - } - - @Override - public void setHasNextLogLine(Boolean hasNextLogLine) { - this.hasNextLogLine = hasNextLogLine; - } - - @Override - public List consumeResultContent() { - List ret = new LinkedList<>(); - resultContent.drainTo(ret, resultContent.size()); - return ret; - } - - public void appendResultContent(LinkisResultSet resultContent) { - this.resultContent.add(resultContent); - } - - @Override - public Boolean hasNextResultPage() { - return hasNextResultPage; - } - - public void setHasNextResultPage(Boolean hasNextResultPage) { - this.hasNextResultPage = hasNextResultPage; - } - - @Override - public void sendLogFin() { - if (this.logFinevent != null && this.logFinevent.isRegistered()) { - this.logFinevent.notifyObserver(resultFinEvent, null); - } - } - - @Override - public boolean logFinReceived() { - return this.logFinListener.isTriggered(); - } - - @Override - public void sendResultFin() { - if (this.resultFinEvent != null && this.resultFinEvent.isRegistered()) { - this.resultFinEvent.notifyObserver(resultFinEvent, null); - } - } - - @Override - public boolean resultFinReceived() { - return this.resultFinListener.isTriggered(); - } - - @Override - public boolean hasResult() { - return hasResult; - } - - @Override - public void setHasResult(boolean hasResult) { - this.hasResult = hasResult; - } - - @Override - public boolean isSuccess() { - return success; - } - - @Override - public void setSuccess(boolean success) { - this.success = success; - } - - public String getInstance() { - return instance; - } - - public void setInstance(String instance) { - this.instance = instance; - } - - public String getUmUser() { - return umUser; - } - - public void setUmUser(String umUser) { - this.umUser = umUser; - } - - public String getSimpleExecId() { - return simpleExecId; - } - - public void setSimpleExecId(String simpleExecId) { - this.simpleExecId = simpleExecId; - } - - public String getExecutionCode() { - return executionCode; - } - - public void setExecutionCode(String executionCode) { - this.executionCode = executionCode; - } - - public String getEngineType() { - return engineType; - } - - public void setEngineType(String engineType) { - this.engineType = engineType; - } - - public String getRunType() { - return runType; - } - - public void setRunType(String runType) { - this.runType = runType; - } - - public Long getCostTime() { - return costTime; - } - - public void setCostTime(Long costTime) { - this.costTime = costTime; - } - - public Date getCreatedTime() { - return createdTime; - } - - public void setCreatedTime(Date createdTime) { - this.createdTime = createdTime; - } - - public Date getUpdatedTime() { - return updatedTime; - } - - public void setUpdatedTime(Date updatedTime) { - this.updatedTime = updatedTime; - } - - public Date getEngineStartTime() { - return engineStartTime; - } - - public void setEngineStartTime(Date engineStartTime) { - this.engineStartTime = engineStartTime; - } - - public String getExecuteApplicationName() { - return executeApplicationName; - } - - public void setExecuteApplicationName(String executeApplicationName) { - this.executeApplicationName = executeApplicationName; - } - - public String getRequestApplicationName() { - return requestApplicationName; - } - - public void setRequestApplicationName(String requestApplicationName) { - this.requestApplicationName = requestApplicationName; - } - - @Override - public void updateByOperResult(LinkisOperResultAdapter adapter) { - if (adapter.getJobID() != null) { - setJobId(adapter.getJobID()); - } - if (adapter.getUser() != null) { - setUser(adapter.getUser()); - } - if (adapter.getJobStatus() != null) { - setJobStatus(adapter.getJobStatus()); - } - if (adapter.getStrongerExecId() != null) { - setExecID(adapter.getStrongerExecId()); - } - if (adapter.getJobProgress() != null) { - setJobProgress(adapter.getJobProgress()); - } - if (adapter.getLogPath() != null) { - setLogPath(adapter.getLogPath()); - } - if (adapter.getResultLocation() != null) { - setResultLocation(adapter.getResultLocation()); - } - if (adapter.getResultSetPaths() != null) { - setResultSetPaths(adapter.getResultSetPaths()); - } - if (adapter.getErrCode() != null) { - setErrCode(adapter.getErrCode()); - } - if (adapter.getErrDesc() != null) { - setErrDesc(adapter.getErrDesc()); - } - if (adapter.getLog() != null - && adapter.getNextLogLine() != null - && adapter.hasNextLogLine() != null) { - setNextLogLineIdx(adapter.getNextLogLine()); - setHasNextLogLine(adapter.hasNextLogLine()); - appendLog(adapter.getLog()); - } - if (adapter.getResultContent() != null && adapter.resultHasNextPage() != null) { - setHasNextResultPage(adapter.resultHasNextPage()); - appendResultContent(adapter.getResultContent()); - } - if (adapter.getInstance() != null) { - setInstance(adapter.getInstance()); - } - if (adapter.getUmUser() != null) { - setUmUser(adapter.getUmUser()); - } - if (adapter.getSimpleExecId() != null) { - setSimpleExecId(adapter.getSimpleExecId()); - } - if (adapter.getExecutionCode() != null) { - setExecutionCode(adapter.getExecutionCode()); - } - if (adapter.getEngineType() != null) { - setEngineType(adapter.getEngineType()); - } - if (adapter.getRunType() != null) { - setRunType(adapter.getRunType()); - } - if (adapter.getCostTime() != null) { - setCostTime(adapter.getCostTime()); - } - if (adapter.getCreatedTime() != null) { - setCreatedTime(adapter.getCreatedTime()); - } - if (adapter.getUpdatedTime() != null) { - setUpdatedTime(adapter.getUpdatedTime()); - } - if (adapter.getEngineStartTime() != null) { - setEngineStartTime(adapter.getEngineStartTime()); - } - if (adapter.getExecuteApplicationName() != null) { - setExecuteApplicationName(adapter.getExecuteApplicationName()); - } - if (adapter.getRequestApplicationName() != null) { - setRequestApplicationName(adapter.getRequestApplicationName()); - } - } - - @Override - public LinkisJobDataImpl clone() throws CloneNotSupportedException { - LinkisJobDataImpl ret = (LinkisJobDataImpl) super.clone(); - if (logBuffer != null) { - ret.logBuffer = new LinkedBlockingDeque(this.logBuffer); - } - if (this.resultContent != null) { - ret.resultContent = new LinkedBlockingDeque<>(); - for (LinkisResultSet r1 : resultContent) { - ret.resultContent.add(r1.clone()); - } - } - if (this.resultSetPaths != null) { - ret.setResultSetPaths(Arrays.copyOf(this.resultSetPaths, this.resultSetPaths.length)); - } - /* - These be shared and hence should not be deep copied. - */ - ret.logFinevent = this.logFinevent; - ret.logFinListener = this.logFinListener; - ret.resultFinEvent = this.resultFinEvent; - ret.resultFinListener = this.resultFinListener; - - return ret; - } -} diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/interactor/job/data/LinkisLogData.java b/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/interactor/job/data/LinkisLogData.java deleted file mode 100644 index 3df7cc5a47..0000000000 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/interactor/job/data/LinkisLogData.java +++ /dev/null @@ -1,51 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.cli.application.interactor.job.data; - -import org.apache.linkis.cli.application.observer.listener.LinkisClientListener; - -public interface LinkisLogData extends LinkisJobData, Cloneable { - - void notifyLogListener(); - - void registerincLogListener(LinkisClientListener observer); - - String getLogPath(); - - Integer getNextLogLineIdx(); - - void setHasNextLogLine(Boolean hasNextLog); - - Boolean hasNextLogLine(); - - String consumeLog(); - - boolean isIncLogMode(); - - /* - incLogMode = true: for sync-submission, wait for job complete while get incremental log - incLogMode = false: for async-submission, output all log we have currently - */ - void setIncLogMode(boolean incLogMode); - - void sendLogFin(); - - boolean logFinReceived(); - - LinkisLogData clone() throws CloneNotSupportedException; -} diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/interactor/job/data/LinkisOnceJobData.java b/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/interactor/job/data/LinkisOnceJobData.java deleted file mode 100644 index 89130b62af..0000000000 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/interactor/job/data/LinkisOnceJobData.java +++ /dev/null @@ -1,242 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.cli.application.interactor.job.data; - -import org.apache.linkis.cli.application.observer.listener.LinkisClientListener; -import org.apache.linkis.cli.application.operator.ujes.LinkisOperResultAdapter; -import org.apache.linkis.cli.common.entity.job.JobStatus; - -import java.util.List; - -public class LinkisOnceJobData implements LinkisJobData, LinkisLogData, LinkisResultData { - - private SimpleOnceJobAdapter onceJobAdapter; - - private boolean hasResult = true; - - public SimpleOnceJobAdapter getOnceJobAdapter() { - return onceJobAdapter; - } - - public void setOnceJobAdapter(SimpleOnceJobAdapter onceJobAdapter) { - this.onceJobAdapter = onceJobAdapter; - } - - @Override - public void registerincLogListener(LinkisClientListener observer) { - onceJobAdapter.registerincLogListener(observer); - } - - @Override - public void notifyLogListener() { - onceJobAdapter.notifyLogListener(); - } - - @Override - public boolean isIncLogMode() { - return onceJobAdapter.isIncLogMode(); - } - - @Override - public void setIncLogMode(boolean incLogMode) { - onceJobAdapter.setIncLogMode(incLogMode); - } - - @Override - public String consumeLog() { - return onceJobAdapter.consumeLog(); - } - - public void appendLog(String log) { - onceJobAdapter.appendLog(log); - } - - @Override - public final String getLogPath() { - return onceJobAdapter.getLogPath(); - } - - public final void setLogPath(String logPath) { - onceJobAdapter.setLogPath(logPath); - } - - @Override - public Integer getNextLogLineIdx() { - return onceJobAdapter.getNextLogLineIdx(); - } - - public void setNextLogLineIdx(Integer nextLogLineIdx) { - onceJobAdapter.setNextLogLineIdx(nextLogLineIdx); - } - - @Override - public Boolean hasNextLogLine() { - return onceJobAdapter.hasNextLogLine(); - } - - @Override - public void setHasNextLogLine(Boolean hasNextLogLine) { - onceJobAdapter.setHasNextLogLine(hasNextLogLine); - } - - @Override - public List consumeResultContent() { - return onceJobAdapter.consumeResultContent(); - } - - public void appendResultContent(LinkisResultSet resultContent) { - onceJobAdapter.appendResultContent(resultContent); - } - - @Override - public Boolean hasNextResultPage() { - return onceJobAdapter.hasNextResultPage(); - } - - public void setHasNextResultPage(Boolean hasNextResultPage) { - onceJobAdapter.setHasNextResultPage(hasNextResultPage); - } - - @Override - public final String getResultLocation() { - return onceJobAdapter.getResultLocation(); - } - - public final void setResultLocation(String resultLocation) { - onceJobAdapter.setResultLocation(resultLocation); - } - - @Override - public String[] getResultSetPaths() { - return onceJobAdapter.getResultSetPaths(); - } - - public final void setResultSetPaths(String[] resultSetPaths) { - onceJobAdapter.setResultSetPaths(resultSetPaths); - } - - @Override - public void sendLogFin() { - onceJobAdapter.sendLogFin(); - } - - @Override - public boolean logFinReceived() { - return onceJobAdapter.logFinReceived(); - } - - @Override - public void sendResultFin() { - onceJobAdapter.sendResultFin(); - } - - @Override - public boolean resultFinReceived() { - return onceJobAdapter.resultFinReceived(); - } - - @Override - public boolean hasResult() { - return hasResult; - } - - @Override - public void setHasResult(boolean hasResult) { - this.hasResult = hasResult; - } - - @Override - public JobStatus getJobStatus() { - return onceJobAdapter.getJobStatus(); - } - - public void setJobStatus(JobStatus jobStatus) { - onceJobAdapter.setJobStatus(jobStatus); - } - - @Override - public String getJobID() { - return onceJobAdapter.getJobID(); - } - - @Override - public String getUser() { - return onceJobAdapter.getUser(); - } - - @Override - public String getMessage() { - return onceJobAdapter.getMessage(); - } - - @Override - public void setMessage(String message) { - onceJobAdapter.setMessage(message); - } - - @Override - public Exception getException() { - return onceJobAdapter.getException(); - } - - @Override - public void setException(Exception e) { - onceJobAdapter.setException(e); - } - - @Override - public String getExecID() { - return onceJobAdapter.getJobID(); - } // No Need - - @Override - public float getJobProgress() { - return 0; - } - - @Override - public Integer getErrCode() { - return onceJobAdapter.getErrCode(); - } - - @Override - public String getErrDesc() { - return onceJobAdapter.getErrDesc(); - } - - @Override - public boolean isSuccess() { - return onceJobAdapter.isSuccess(); - } - - @Override - public void setSuccess(boolean success) { - onceJobAdapter.setSuccess(success); - } - - @Override - public void updateByOperResult(LinkisOperResultAdapter adapter) { - // No need - } - - @Override - public LinkisOnceJobData clone() throws CloneNotSupportedException { - throw new CloneNotSupportedException(); - // return null; - } -} diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/interactor/job/data/LinkisResultData.java b/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/interactor/job/data/LinkisResultData.java deleted file mode 100644 index f8e61a790d..0000000000 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/interactor/job/data/LinkisResultData.java +++ /dev/null @@ -1,39 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.cli.application.interactor.job.data; - -import java.util.List; - -public interface LinkisResultData extends LinkisJobData, Cloneable { - - String getResultLocation(); - - String[] getResultSetPaths(); - - Boolean hasNextResultPage(); - - List consumeResultContent(); - - void sendResultFin(); - - boolean resultFinReceived(); - - boolean hasResult(); - - void setHasResult(boolean hasResult); -} diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/interactor/job/data/SimpleOnceJobAdapter.java b/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/interactor/job/data/SimpleOnceJobAdapter.java deleted file mode 100644 index 69a76d6240..0000000000 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/interactor/job/data/SimpleOnceJobAdapter.java +++ /dev/null @@ -1,377 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.cli.application.interactor.job.data; - -import org.apache.linkis.cli.application.constants.AppConstants; -import org.apache.linkis.cli.application.constants.AppKeys; -import org.apache.linkis.cli.application.constants.LinkisKeys; -import org.apache.linkis.cli.application.interactor.job.LinkisJobStatus; -import org.apache.linkis.cli.application.interactor.job.desc.LinkisOnceDesc; -import org.apache.linkis.cli.application.observer.event.LinkisClientEvent; -import org.apache.linkis.cli.application.observer.event.LogStartEvent; -import org.apache.linkis.cli.application.observer.event.TriggerEvent; -import org.apache.linkis.cli.application.observer.listener.LinkisClientListener; -import org.apache.linkis.cli.application.observer.listener.TriggerEventListener; -import org.apache.linkis.cli.application.operator.once.OnceJobConstants; -import org.apache.linkis.cli.application.operator.ujes.LinkisOperResultAdapter; -import org.apache.linkis.cli.application.operator.ujes.UJESClientFactory; -import org.apache.linkis.cli.common.entity.job.JobStatus; -import org.apache.linkis.cli.common.entity.var.VarAccess; -import org.apache.linkis.cli.common.exception.error.ErrorLevel; -import org.apache.linkis.cli.core.exception.LinkisClientExecutionException; -import org.apache.linkis.cli.core.exception.error.CommonErrMsg; -import org.apache.linkis.computation.client.LinkisJobBuilder$; -import org.apache.linkis.computation.client.once.simple.SimpleOnceJob; -import org.apache.linkis.computation.client.once.simple.SimpleOnceJobBuilder; -import org.apache.linkis.computation.client.once.simple.SubmittableSimpleOnceJob; -import org.apache.linkis.computation.client.operator.impl.EngineConnLogOperator; -import org.apache.linkis.computation.client.operator.impl.EngineConnLogs; - -import org.apache.commons.lang3.StringUtils; - -import java.util.LinkedList; -import java.util.List; -import java.util.concurrent.LinkedBlockingDeque; - -public class SimpleOnceJobAdapter implements LinkisLogData { - LinkisJobStatus jobStatus = LinkisJobStatus.UNSUBMITTED; - EngineConnLogOperator logOperator = null; - private String serverUrl; - private SimpleOnceJob onceJob; - private String engineTypeForECM; - private String message; - private Exception exception; - private boolean success; - private Boolean incLogMode; - private LinkedBlockingDeque logBuffer = new LinkedBlockingDeque(); - // private String logPath; // remote path for job log - // private Integer nextLogLineIdx = 0; - private Boolean hasNextLogLine = true; - private String resultLocation; - private String[] resultSetPaths = null; // remote paths for job result set - private LinkedBlockingDeque resultContent = new LinkedBlockingDeque<>(); - private Boolean hasNextResultPage; - private LinkisClientEvent logstartEvent = new LogStartEvent(); - private TriggerEvent logFinEvent = new TriggerEvent(); - private TriggerEventListener logFinListener = new TriggerEventListener(); - private TriggerEvent resultFinEvent = new TriggerEvent(); - private TriggerEventListener resultFinListener = new TriggerEventListener(); - - { - logFinEvent.register(logFinListener); - resultFinEvent.register(resultFinListener); - } - - public void init(LinkisOnceDesc desc) { - VarAccess stdVarAccess = desc.getStdVarAccess(); - VarAccess sysVarAccess = desc.getSysVarAccess(); - - serverUrl = stdVarAccess.getVar(String.class, AppKeys.LINKIS_COMMON_GATEWAY_URL); - - LinkisJobBuilder$.MODULE$.setDefaultClientConfig( - UJESClientFactory.generateDWSClientConfig(stdVarAccess, sysVarAccess)); - LinkisJobBuilder$.MODULE$.setDefaultUJESClient( - UJESClientFactory.getReusable(stdVarAccess, sysVarAccess)); - - String engineTypeRaw = (String) desc.getLabelMap().get(LinkisKeys.KEY_ENGINETYPE); - engineTypeForECM = engineTypeRaw; - - if (StringUtils.isNotBlank(engineTypeRaw)) { - engineTypeForECM = StringUtils.split(engineTypeRaw, "-")[0]; - } else { - engineTypeForECM = ""; - } // TODO: remove parsing and let server side parse engineType - - onceJob = - new SimpleOnceJobBuilder() - .setCreateService(AppConstants.LINKIS_CLI) - .addExecuteUser(desc.getProxyUser()) - .setStartupParams(desc.getParamConfMap()) - .setLabels(desc.getLabelMap()) - .setRuntimeParams(desc.getParamRunTimeMap()) - .setSource(desc.getSourceMap()) - .setVariableMap(desc.getParamVarsMap()) - .setJobContent(desc.getJobContentMap()) - .build(); - } - - public String getServerUrl() { - return serverUrl; - } - - public SimpleOnceJob getOnceJob() { - return onceJob; - } - - public void setOnceJob(SimpleOnceJob onceJob) { - this.onceJob = onceJob; - } - - private void panicIfNull(Object obj) { - if (obj == null) { - throw new LinkisClientExecutionException( - "EXE0040", - ErrorLevel.ERROR, - CommonErrMsg.ExecutionErr, - "Instance of " + obj.getClass().getCanonicalName() + " is null"); - } - } - - public void submit() { - panicIfNull(onceJob); - if (!(onceJob instanceof SubmittableSimpleOnceJob)) { - throw new LinkisClientExecutionException( - "EXE0041", - ErrorLevel.ERROR, - CommonErrMsg.ExecutionErr, - "onceJob is not properly initiated"); - } - ((SubmittableSimpleOnceJob) onceJob).submit(); - } - - public void kill() { - panicIfNull(onceJob); - onceJob.kill(); - } - - public String getJobID() { - return onceJob.getId(); - } - - @Override - public String getUser() { - return "TODO"; - } - - public void updateStatus() { - panicIfNull(onceJob); - String status = onceJob.getStatus(); - panicIfNull(status); - jobStatus = LinkisJobStatus.convertFromNodeStatusString(onceJob.getStatus()); - } - - public LinkisJobStatus getJobStatus() { - return this.jobStatus; - } - - public void setJobStatus(JobStatus jobStatus) { - this.jobStatus = (LinkisJobStatus) jobStatus; - } - - public void waitForComplete() { - panicIfNull(onceJob); - onceJob.waitForCompleted(); - } - - public void queryJobLogOneIteration() { - panicIfNull(onceJob); - updateStatus(); - if (logOperator == null) { - logOperator = - (EngineConnLogOperator) onceJob.getOperator(EngineConnLogOperator.OPERATOR_NAME()); - logOperator.setECMServiceInstance( - ((SubmittableSimpleOnceJob) onceJob).getECMServiceInstance()); - logOperator.setEngineConnType(engineTypeForECM); - // logOperator.setPageSize(OnceJobConstants.MAX_LOG_SIZE_ONCE); - logOperator.setIgnoreKeywords(OnceJobConstants.LOG_IGNORE_KEYWORDS); - } - EngineConnLogs logs = - (EngineConnLogs) logOperator.apply(); // for some reason we have to add type conversion, - // otherwise mvn testCompile fails - StringBuilder logBuilder = new StringBuilder(); - for (String log : logs.logs()) { - logBuilder.append(log).append(System.lineSeparator()); - } - appendLog(logBuilder.toString()); - if ((logs.logs() == null || logs.logs().size() <= 0) && jobStatus.isJobFinishedState()) { - setHasNextLogLine(false); - } - // System.out.println(logs.logs().size()); - } - - public void registerincLogListener(LinkisClientListener observer) { - this.logstartEvent.register(observer); - } - - public void notifyLogListener() { - if (this.logstartEvent.isRegistered()) { - logstartEvent.notifyObserver(logstartEvent, this); - } - } - - public boolean isIncLogMode() { - return this.incLogMode; - } - - public void setIncLogMode(boolean incLogMode) { - this.incLogMode = incLogMode; - } - - public String consumeLog() { - List logs = new LinkedList<>(); - this.logBuffer.drainTo(logs, this.logBuffer.size()); - StringBuilder tmp = new StringBuilder(); - for (String str : logs) { - tmp.append(str); - } - return tmp.toString(); - } - - public void appendLog(String log) { - this.logBuffer.add(log); - } - - public final String getLogPath() { - return null; - } - - public final void setLogPath(String logPath) { - return; - } - - public Integer getNextLogLineIdx() { - return null; - } - - public void setNextLogLineIdx(Integer nextLogLineIdx) { - return; - } - - public Boolean hasNextLogLine() { - return hasNextLogLine; - } - - public void setHasNextLogLine(Boolean hasNextLogLine) { - this.hasNextLogLine = hasNextLogLine; - } - - public List consumeResultContent() { - List ret = new LinkedList<>(); - resultContent.drainTo(ret, resultContent.size()); - return ret; - } - - public void appendResultContent(LinkisResultSet resultContent) { - this.resultContent.add(resultContent); - } - - public Boolean hasNextResultPage() { - return hasNextResultPage; - } - - public void setHasNextResultPage(Boolean hasNextResultPage) { - this.hasNextResultPage = hasNextResultPage; - } - - public final String getResultLocation() { - return resultLocation; - } - - public final void setResultLocation(String resultLocation) { - this.resultLocation = resultLocation; - } - - public String[] getResultSetPaths() { - return resultSetPaths; - } - - public final void setResultSetPaths(String[] resultSetPaths) { - this.resultSetPaths = resultSetPaths; - } - - public void sendLogFin() { - if (this.logFinEvent != null && this.logFinEvent.isRegistered()) { - this.logFinEvent.notifyObserver(resultFinEvent, null); - } - } - - public boolean logFinReceived() { - return this.logFinListener.isTriggered(); - } - - public void sendResultFin() { - if (this.resultFinEvent != null && this.resultFinEvent.isRegistered()) { - this.resultFinEvent.notifyObserver(resultFinEvent, null); - } - } - - public boolean resultFinReceived() { - return this.resultFinListener.isTriggered(); - } - - @Override - public String getMessage() { - return message; - } - - @Override - public void setMessage(String message) { - this.message = message; - } - - @Override - public Exception getException() { - return exception; - } - - @Override - public void setException(Exception e) { - this.exception = e; - } - - @Override - public String getExecID() { - return getJobID(); - } // No Need - - @Override - public float getJobProgress() { - return 0; - } - - @Override - public Integer getErrCode() { - return null; - } - - @Override - public String getErrDesc() { - return null; - } - - @Override - public boolean isSuccess() { - return success; - } - - @Override - public void setSuccess(boolean success) { - this.success = success; - } - - @Override - public void updateByOperResult(LinkisOperResultAdapter adapter) { - // No need - } - - @Override - public LinkisLogData clone() throws CloneNotSupportedException { - throw new CloneNotSupportedException(); - // return null; - } -} diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/interactor/job/subtype/LinkisSubmitSubType.java b/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/interactor/job/subtype/LinkisSubmitSubType.java deleted file mode 100644 index 49d3d5ea91..0000000000 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/interactor/job/subtype/LinkisSubmitSubType.java +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.cli.application.interactor.job.subtype; - -import org.apache.linkis.cli.common.entity.job.JobSubType; - -public enum LinkisSubmitSubType implements JobSubType { - SUBMIT("submit"); - - private String name; - - LinkisSubmitSubType(String name) { - this.name = name; - } - - @Override - public String getName() { - return name; - } -} diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/observer/listener/TriggerEventListener.java b/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/observer/listener/TriggerEventListener.java deleted file mode 100644 index 8811441d02..0000000000 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/observer/listener/TriggerEventListener.java +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.cli.application.observer.listener; - -import org.apache.linkis.cli.application.observer.event.LinkisClientEvent; - -import java.util.concurrent.atomic.AtomicBoolean; - -public class TriggerEventListener implements LinkisClientListener { - AtomicBoolean atomicFlag = new AtomicBoolean(false); - - @Override - public void update(LinkisClientEvent event, Object msg) { - atomicFlag.compareAndSet(false, true); - } - - public Boolean isTriggered() { - return atomicFlag.get(); - } -} diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/operator/OperatorUtils.java b/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/operator/OperatorUtils.java deleted file mode 100644 index 767d44aea1..0000000000 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/operator/OperatorUtils.java +++ /dev/null @@ -1,70 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.cli.application.operator; - -public class OperatorUtils { - public static int getNumOfLines(String str) { - if (str == null || str.length() == 0) { - return 0; - } - int lines = 1; - int len = str.length(); - for (int pos = 0; pos < len; pos++) { - char c = str.charAt(pos); - if (c == '\r') { - lines++; - if (pos + 1 < len && str.charAt(pos + 1) == '\n') { - pos++; - } - } else if (c == '\n') { - lines++; - } - } - return lines; - } - - public static int getFirstIndexSkippingLines(String str, Integer lines) { - if (str == null || str.length() == 0 || lines < 0) { - return -1; - } - if (lines == 0) { - return 0; - } - - int curLineIdx = 0; - int len = str.length(); - for (int pos = 0; pos < len; pos++) { - char c = str.charAt(pos); - if (c == '\r') { - curLineIdx++; - if (pos + 1 < len && str.charAt(pos + 1) == '\n') { - pos++; - } - } else if (c == '\n') { - curLineIdx++; - } else { - continue; - } - - if (curLineIdx >= lines) { - return pos + 1; - } - } - return -1; - } -} diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/present/LinkisResultInfoPresenter.java b/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/present/LinkisResultInfoPresenter.java deleted file mode 100644 index 5a953fa5d7..0000000000 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/present/LinkisResultInfoPresenter.java +++ /dev/null @@ -1,114 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.cli.application.present; - -import org.apache.linkis.cli.application.present.model.LinkisResultInfoModel; -import org.apache.linkis.cli.common.entity.present.Model; -import org.apache.linkis.cli.common.entity.present.PresentWay; -import org.apache.linkis.cli.common.entity.present.Presenter; -import org.apache.linkis.cli.common.exception.error.ErrorLevel; -import org.apache.linkis.cli.core.exception.PresenterException; -import org.apache.linkis.cli.core.exception.error.CommonErrMsg; -import org.apache.linkis.cli.core.present.PresentModeImpl; -import org.apache.linkis.cli.core.present.display.DisplayOperFactory; -import org.apache.linkis.cli.core.present.display.data.StdoutDisplayData; -import org.apache.linkis.cli.core.utils.LogUtils; - -import org.apache.commons.lang3.StringUtils; - -public class LinkisResultInfoPresenter implements Presenter { - @Override - public void present(Model model, PresentWay presentWay) { - if (!(model instanceof LinkisResultInfoModel)) { - throw new PresenterException( - "PST0001", - ErrorLevel.ERROR, - CommonErrMsg.PresenterErr, - "Input model for \"LinkisResultInfoPresenter\" is not instance of \"LinkisResultInfoModel\""); - } - DisplayOperFactory.getDisplayOper(PresentModeImpl.STDOUT) - .doOutput(new StdoutDisplayData(formatResultIndicator((LinkisResultInfoModel) model))); - } - - protected String formatResultIndicator(LinkisResultInfoModel model) { - StringBuilder infoBuilder = new StringBuilder(); - String extraMsgStr = ""; - - if (model.getMessage() != null) { - extraMsgStr = model.getMessage().toString(); - } - if (model.getJobStatus().isJobSuccess()) { - - LogUtils.getInformationLogger().info("Job execute successfully! Will try get execute result"); - infoBuilder - .append("============Result:================") - .append(System.lineSeparator()) - .append("TaskId:") - .append(model.getJobID()) - .append(System.lineSeparator()) - .append("ExecId: ") - .append(model.getExecID()) - .append(System.lineSeparator()) - .append("User:") - .append(model.getUser()) - .append(System.lineSeparator()) - .append("Current job status:") - .append(model.getJobStatus()) - .append(System.lineSeparator()) - .append("extraMsg: ") - .append(extraMsgStr) - .append(System.lineSeparator()) - .append("result: ") - .append(extraMsgStr) - .append(System.lineSeparator()); - } else if (model.getJobStatus().isJobFinishedState()) { - LogUtils.getInformationLogger().info("Job failed! Will not try get execute result."); - infoBuilder - .append("============Result:================") - .append(System.lineSeparator()) - .append("TaskId:") - .append(model.getJobID()) - .append(System.lineSeparator()) - .append("ExecId: ") - .append(model.getExecID()) - .append(System.lineSeparator()) - .append("User:") - .append(model.getUser()) - .append(System.lineSeparator()) - .append("Current job status:") - .append(model.getJobStatus()) - .append(System.lineSeparator()) - .append("extraMsg: ") - .append(extraMsgStr) - .append(System.lineSeparator()); - if (model.getErrCode() != null) { - infoBuilder.append("errCode: ").append(model.getErrCode()).append(System.lineSeparator()); - } - if (StringUtils.isNotBlank(model.getErrDesc())) { - infoBuilder.append("errDesc: ").append(model.getErrDesc()).append(System.lineSeparator()); - } - } else { - throw new PresenterException( - "PST0011", - ErrorLevel.ERROR, - CommonErrMsg.PresenterErr, - "Job is not completed but triggered ResultPresenter"); - } - return infoBuilder.toString(); - } -} diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/present/LinkisResultPresenter.java b/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/present/LinkisResultPresenter.java deleted file mode 100644 index b5e39c4e2e..0000000000 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/present/LinkisResultPresenter.java +++ /dev/null @@ -1,232 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.cli.application.present; - -import org.apache.linkis.cli.application.constants.AppConstants; -import org.apache.linkis.cli.application.interactor.job.data.LinkisResultSet; -import org.apache.linkis.cli.application.present.model.LinkisResultModel; -import org.apache.linkis.cli.common.entity.present.Model; -import org.apache.linkis.cli.common.entity.present.PresentWay; -import org.apache.linkis.cli.common.entity.present.Presenter; -import org.apache.linkis.cli.common.exception.error.ErrorLevel; -import org.apache.linkis.cli.core.exception.PresenterException; -import org.apache.linkis.cli.core.exception.error.CommonErrMsg; -import org.apache.linkis.cli.core.present.PresentModeImpl; -import org.apache.linkis.cli.core.present.PresentWayImpl; -import org.apache.linkis.cli.core.present.display.DisplayOperFactory; -import org.apache.linkis.cli.core.present.display.DisplayOperator; -import org.apache.linkis.cli.core.present.display.data.FileDisplayData; -import org.apache.linkis.cli.core.utils.CommonUtils; -import org.apache.linkis.cli.core.utils.LogUtils; - -import org.apache.commons.lang3.StringUtils; - -import java.text.MessageFormat; -import java.util.*; - -public class LinkisResultPresenter implements Presenter { - - @Override - public void present(Model model, PresentWay presentWay) { - if (!(model instanceof LinkisResultModel)) { - throw new PresenterException( - "PST0001", - ErrorLevel.ERROR, - CommonErrMsg.PresenterErr, - "Input model for \"LinkisResultPresenter\" is not instance of \"LinkisResultModel\""); - } - if (!(presentWay instanceof PresentWayImpl)) { - throw new PresenterException( - "PST0002", - ErrorLevel.ERROR, - CommonErrMsg.PresenterErr, - "Input PresentWay for \"LinkisResultPresenter\" is not instance of \"PresentWayImpl\""); - } - LinkisResultModel resultModel = (LinkisResultModel) model; - PresentWayImpl presentWay1 = (PresentWayImpl) presentWay; - - if (!resultModel.getJobStatus().isJobSuccess()) { - LogUtils.getInformationLogger() - .info("JobStatus is not \'success\'. Will not retrieve result-set."); - return; - } - String msg = ""; - if (resultModel.hasResult()) { - msg = - "Retrieving result-set, may take time if result-set is large, please do not exit program."; - } else { - msg = "Your job has no result."; - } - LogUtils.getInformationLogger().info(msg); - - final DisplayOperator displayOperator = - DisplayOperFactory.getDisplayOper( - presentWay1.getMode()); // currently we don't allow printing log to file here - - int preIdx = -1; - StringBuilder resultSb = new StringBuilder(); - - while (!resultModel.resultFinReceived()) { - preIdx = presentOneIteration(resultModel, preIdx, presentWay1, resultSb, displayOperator); - CommonUtils.doSleepQuietly(500l); - } - presentOneIteration(resultModel, preIdx, presentWay1, resultSb, displayOperator); - - if (presentWay1.getMode() == PresentModeImpl.TEXT_FILE - || StringUtils.isNotBlank(presentWay1.getPath())) { - LogUtils.getInformationLogger() - .info("ResultSet has been successfully written to path: " + presentWay1.getPath()); - } - } - - protected int presentOneIteration( - LinkisResultModel resultModel, - int preIdx, - PresentWayImpl presentWay, - StringBuilder resultSb, - DisplayOperator displayOperator) { - List linkisResultSets = resultModel.consumeResultContent(); - if (linkisResultSets != null && !linkisResultSets.isEmpty()) { - for (LinkisResultSet c : linkisResultSets) { - int idxResultset = c.getResultsetIdx(); - /** - * Notice: we assume result-sets are visited one by one in non-descending order!!! i.e. - * either idxResultset == preIdx or idxResultset - preIdx == 1 i.e. resultsets[0] -> - * resultsets[1] -> ... - */ - if (idxResultset - preIdx != 0 && idxResultset - preIdx != 1) { - throw new PresenterException( - "PST0002", - ErrorLevel.ERROR, - CommonErrMsg.PresenterErr, - "Linkis resultsets are visited in descending order or are not visited one-by-one"); - } - - boolean flag = idxResultset > preIdx; - if (presentWay.isDisplayMetaAndLogo()) { - if (idxResultset - preIdx == 1) { - resultSb.setLength(0); - resultSb - .append(MessageFormat.format(AppConstants.RESULTSET_LOGO, idxResultset + 1)) - .append(System.lineSeparator()); - if (c.getResultMeta() != null) { - resultSb - .append(AppConstants.RESULTSET_META_BEGIN_LOGO) - .append(System.lineSeparator()); - resultSb.append(formatResultMeta(c.getResultMeta())); - resultSb.append(AppConstants.RESULTSET_META_END_LOGO).append(System.lineSeparator()); - } - } - } - preIdx = idxResultset; - String contentStr = formatResultContent(c.getResultMeta(), c.getContent()); - if (contentStr != null) { - resultSb.append(contentStr); - } - if (resultSb.length() != 0) { - String resultFileName = - resultModel.getUser() - + "-task-" - + resultModel.getJobID() - + "-result-" - + String.valueOf(idxResultset + 1) - + ".txt"; - displayOperator.doOutput( - new FileDisplayData(presentWay.getPath(), resultFileName, resultSb.toString(), flag)); - resultSb.setLength(0); - } - } - } - return preIdx; - } - - protected String formatResultMeta(List> metaData) { - - StringBuilder outputBuilder = new StringBuilder(); - - if (metaData == null || metaData.size() == 0) { - return null; - } - - List titles = new ArrayList<>(); - - // gather keys as title - for (LinkedHashMap mapElement : metaData) { - if (mapElement == null || mapElement.size() == 0) { - continue; - } - - Set> entrySet = mapElement.entrySet(); - if (entrySet == null) { - break; - } - for (Map.Entry entry : entrySet) { - String key = entry.getKey(); - if (key != null && !titles.contains(key)) { - titles.add(key); - outputBuilder.append(key).append("\t"); - } - } - } - - outputBuilder.append(System.lineSeparator()); - - // gather value and print to output - for (LinkedHashMap mapElement : metaData) { - if (mapElement == null || mapElement.size() == 0) { - continue; - } - String candidate; - for (String title : titles) { - if (mapElement.containsKey(title)) { - candidate = mapElement.get(title); - } else { - candidate = "NULL"; - } - outputBuilder.append(candidate).append("\t"); - } - outputBuilder.append(System.lineSeparator()); - } - return outputBuilder.toString(); - } - - protected String formatResultContent( - List> metaData, List> contentData) { - - StringBuilder outputBuilder = new StringBuilder(); - if (contentData == null || contentData.size() == 0) { // finished - return null; - } - - int listLen = contentData.size(); - for (int i = 0; i < listLen; i++) { - List listElement = contentData.get(i); - if (listElement == null || listElement.size() == 0) { - continue; - } - for (String element : listElement) { - outputBuilder.append(element).append("\t"); - } - if (i < listLen - 1) { - outputBuilder.append(System.lineSeparator()); - } - } - - return outputBuilder.toString(); - } -} diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/present/model/LinkisJobInfoModel.java b/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/present/model/LinkisJobInfoModel.java deleted file mode 100644 index acfc392e48..0000000000 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/present/model/LinkisJobInfoModel.java +++ /dev/null @@ -1,97 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.cli.application.present.model; - -import org.apache.linkis.cli.application.interactor.job.data.LinkisJobDataImpl; -import org.apache.linkis.cli.common.entity.job.JobStatus; -import org.apache.linkis.cli.common.entity.present.Model; -import org.apache.linkis.cli.common.exception.error.ErrorLevel; -import org.apache.linkis.cli.core.exception.TransformerException; -import org.apache.linkis.cli.core.exception.error.CommonErrMsg; - -import org.apache.commons.lang3.exception.ExceptionUtils; - -import java.util.Date; - -public class LinkisJobInfoModel implements Model { - - private String cid; - private String jobId; - private String message; - private String exception; - private String cause; - - private String taskID; - private String instance; - private String simpleExecId; - private String execId; - private String umUser; - private String executionCode; - private String logPath; - private JobStatus status; - private String engineType; - private String runType; - private Long costTime; - private Date createdTime; - private Date updatedTime; - private Date engineStartTime; - private Integer errCode; - private String errMsg; - private String executeApplicationName; - private String requestApplicationName; - private Float progress; - - @Override - public void buildModel(Object data) { - if (!(data instanceof LinkisJobDataImpl)) { - throw new TransformerException( - "TFM0010", - ErrorLevel.ERROR, - CommonErrMsg.TransformerException, - "Failed to init LinkisJobInfoModel: " - + data.getClass().getCanonicalName() - + "is not instance of \"LinkisJobDataImpl\""); - } - this.jobId = ((LinkisJobDataImpl) data).getJobID(); - this.message = ((LinkisJobDataImpl) data).getMessage(); - this.taskID = ((LinkisJobDataImpl) data).getJobID(); - this.instance = ((LinkisJobDataImpl) data).getInstance(); - this.simpleExecId = ((LinkisJobDataImpl) data).getSimpleExecId(); - this.execId = ((LinkisJobDataImpl) data).getExecID(); - this.umUser = ((LinkisJobDataImpl) data).getUmUser(); - this.executionCode = ((LinkisJobDataImpl) data).getExecutionCode(); - this.logPath = ((LinkisJobDataImpl) data).getLogPath(); - this.status = ((LinkisJobDataImpl) data).getJobStatus(); - this.engineType = ((LinkisJobDataImpl) data).getEngineType(); - this.runType = ((LinkisJobDataImpl) data).getRunType(); - this.costTime = ((LinkisJobDataImpl) data).getCostTime(); - this.createdTime = ((LinkisJobDataImpl) data).getCreatedTime(); - this.updatedTime = ((LinkisJobDataImpl) data).getUpdatedTime(); - this.engineStartTime = ((LinkisJobDataImpl) data).getEngineStartTime(); - this.errCode = ((LinkisJobDataImpl) data).getErrCode(); - this.errMsg = ((LinkisJobDataImpl) data).getErrDesc(); - this.executeApplicationName = ((LinkisJobDataImpl) data).getExecuteApplicationName(); - this.requestApplicationName = ((LinkisJobDataImpl) data).getRequestApplicationName(); - this.progress = ((LinkisJobDataImpl) data).getJobProgress(); - Exception e = ((LinkisJobDataImpl) data).getException(); - if (e != null) { - this.exception = ExceptionUtils.getMessage(e); - this.cause = ExceptionUtils.getRootCauseMessage(e); - } - } -} diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/present/model/LinkisJobKillModel.java b/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/present/model/LinkisJobKillModel.java deleted file mode 100644 index 95eb7d03c2..0000000000 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/present/model/LinkisJobKillModel.java +++ /dev/null @@ -1,63 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.cli.application.present.model; - -import org.apache.linkis.cli.application.interactor.job.data.LinkisJobDataImpl; -import org.apache.linkis.cli.common.entity.job.JobStatus; -import org.apache.linkis.cli.common.entity.present.Model; -import org.apache.linkis.cli.common.exception.error.ErrorLevel; -import org.apache.linkis.cli.core.exception.TransformerException; -import org.apache.linkis.cli.core.exception.error.CommonErrMsg; - -import org.apache.commons.lang3.exception.ExceptionUtils; - -public class LinkisJobKillModel implements Model { - - private String cid; - private String jobId; - private String message; - private String exception; - private String cause; - - private String execID; - private String user; - private JobStatus jobStatus; - - @Override - public void buildModel(Object data) { - if (!(data instanceof LinkisJobDataImpl)) { - throw new TransformerException( - "TFM0010", - ErrorLevel.ERROR, - CommonErrMsg.TransformerException, - "Failed to init LinkisJobKillModel: " - + data.getClass().getCanonicalName() - + "is not instance of \"LinkisJobDataImpl\""); - } - this.jobId = ((LinkisJobDataImpl) data).getJobID(); - this.message = ((LinkisJobDataImpl) data).getMessage(); - this.execID = ((LinkisJobDataImpl) data).getExecID(); - this.user = ((LinkisJobDataImpl) data).getUser(); - this.jobStatus = ((LinkisJobDataImpl) data).getJobStatus(); - Exception e = ((LinkisJobDataImpl) data).getException(); - if (e != null) { - this.exception = ExceptionUtils.getMessage(e); - this.cause = ExceptionUtils.getRootCauseMessage(e); - } - } -} diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/present/model/LinkisResultInfoModel.java b/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/present/model/LinkisResultInfoModel.java deleted file mode 100644 index bfa0ae6cc8..0000000000 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/present/model/LinkisResultInfoModel.java +++ /dev/null @@ -1,84 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.cli.application.present.model; - -import org.apache.linkis.cli.application.interactor.job.data.LinkisResultData; -import org.apache.linkis.cli.common.entity.job.JobStatus; -import org.apache.linkis.cli.common.entity.present.Model; -import org.apache.linkis.cli.common.exception.error.ErrorLevel; -import org.apache.linkis.cli.core.exception.TransformerException; -import org.apache.linkis.cli.core.exception.error.CommonErrMsg; - -public class LinkisResultInfoModel implements Model { - private String jobID; - private String execID; - private String user; - private JobStatus jobStatus; - private String message; - private Integer errCode; - private String errDesc; - - @Override - public void buildModel(Object data) { - if (!(data instanceof LinkisResultData)) { - throw new TransformerException( - "TFM0010", - ErrorLevel.ERROR, - CommonErrMsg.TransformerException, - "Failed to init LinkisResultInfoModel: " - + data.getClass().getCanonicalName() - + "is not instance of \"LinkisResultData\""); - } - LinkisResultData jobData = (LinkisResultData) data; - jobID = jobData.getJobID(); - execID = jobData.getExecID(); - user = jobData.getUser(); - jobStatus = jobData.getJobStatus(); - message = jobData.getMessage(); - errCode = jobData.getErrCode(); - errDesc = jobData.getErrDesc(); - } - - public String getJobID() { - return jobID; - } - - public String getExecID() { - return execID; - } - - public String getUser() { - return user; - } - - public JobStatus getJobStatus() { - return jobStatus; - } - - public String getMessage() { - return message; - } - - public Integer getErrCode() { - return errCode; - } - - public String getErrDesc() { - return errDesc; - } -} diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/present/model/LinkisSubmitResultModel.java b/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/present/model/LinkisSubmitResultModel.java deleted file mode 100644 index 6b622e1453..0000000000 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/present/model/LinkisSubmitResultModel.java +++ /dev/null @@ -1,56 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.cli.application.present.model; - -import org.apache.linkis.cli.application.interactor.job.data.LinkisJobDataImpl; -import org.apache.linkis.cli.common.entity.job.JobStatus; -import org.apache.linkis.cli.common.entity.present.Model; -import org.apache.linkis.cli.common.exception.error.ErrorLevel; -import org.apache.linkis.cli.core.exception.TransformerException; -import org.apache.linkis.cli.core.exception.error.CommonErrMsg; - -import org.apache.commons.lang3.exception.ExceptionUtils; - -public class LinkisSubmitResultModel implements Model { - private String jobId; - private JobStatus status; - private String message; - private String exception; - private String cause; - - @Override - public void buildModel(Object data) { - if (!(data instanceof LinkisJobDataImpl)) { - throw new TransformerException( - "TFM0010", - ErrorLevel.ERROR, - CommonErrMsg.TransformerException, - "Failed to init LinkisJobInfoModel: " - + data.getClass().getCanonicalName() - + "is not instance of \"LinkisJobDataImpl\""); - } - this.jobId = ((LinkisJobDataImpl) data).getJobID(); - this.status = ((LinkisJobDataImpl) data).getJobStatus(); - this.message = ((LinkisJobDataImpl) data).getMessage(); - Exception e = ((LinkisJobDataImpl) data).getException(); - if (e != null) { - this.exception = ExceptionUtils.getMessage(e); - this.cause = ExceptionUtils.getRootCauseMessage(e); - } - } -} diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/suite/ExecutionSuite.java b/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/suite/ExecutionSuite.java deleted file mode 100644 index 8b05a7cf78..0000000000 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/suite/ExecutionSuite.java +++ /dev/null @@ -1,61 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.cli.application.suite; - -import org.apache.linkis.cli.common.entity.execution.Execution; -import org.apache.linkis.cli.common.entity.job.Job; -import org.apache.linkis.cli.common.entity.result.ResultHandler; - -import java.util.Map; - -public class ExecutionSuite { - Execution execution; - Map jobs; - ResultHandler[] resultHandlers; - - public ExecutionSuite( - Execution execution, Map jobs, ResultHandler... resultHandlers) { - this.execution = execution; - this.jobs = jobs; - this.resultHandlers = resultHandlers; - } - - public Execution getExecution() { - return execution; - } - - public void setExecution(Execution execution) { - this.execution = execution; - } - - public Map getJobs() { - return jobs; - } - - public void setJobs(Map jobs) { - this.jobs = jobs; - } - - public ResultHandler[] getResultHandlers() { - return resultHandlers; - } - - public void setResultHandlers(ResultHandler[] resultHandlers) { - this.resultHandlers = resultHandlers; - } -} diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/suite/ExecutionSuiteFactory.java b/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/suite/ExecutionSuiteFactory.java deleted file mode 100644 index c55ee7b9e4..0000000000 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/suite/ExecutionSuiteFactory.java +++ /dev/null @@ -1,225 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.cli.application.suite; - -import org.apache.linkis.cli.application.constants.AppConstants; -import org.apache.linkis.cli.application.constants.AppKeys; -import org.apache.linkis.cli.application.interactor.command.LinkisCmdType; -import org.apache.linkis.cli.application.interactor.job.builder.LinkisManageJobBuilder; -import org.apache.linkis.cli.application.interactor.job.builder.LinkisOnceJobBuilder; -import org.apache.linkis.cli.application.interactor.job.builder.LinkisSubmitJobBuilder; -import org.apache.linkis.cli.application.interactor.job.subtype.LinkisManSubType; -import org.apache.linkis.cli.application.interactor.job.subtype.LinkisSubmitSubType; -import org.apache.linkis.cli.application.interactor.validate.LinkisManageValidator; -import org.apache.linkis.cli.application.interactor.validate.LinkisOnceSubmitValidator; -import org.apache.linkis.cli.application.interactor.validate.LinkisSubmitValidator; -import org.apache.linkis.cli.application.present.LinkisLogPresenter; -import org.apache.linkis.cli.application.present.LinkisResultInfoPresenter; -import org.apache.linkis.cli.application.present.LinkisResultPresenter; -import org.apache.linkis.cli.application.present.model.LinkisJobInfoModel; -import org.apache.linkis.cli.application.present.model.LinkisJobKillModel; -import org.apache.linkis.cli.application.present.model.LinkisResultInfoModel; -import org.apache.linkis.cli.application.present.model.LinkisResultModel; -import org.apache.linkis.cli.application.utils.Utils; -import org.apache.linkis.cli.common.entity.command.CmdType; -import org.apache.linkis.cli.common.entity.execution.Execution; -import org.apache.linkis.cli.common.entity.job.Job; -import org.apache.linkis.cli.common.entity.job.JobData; -import org.apache.linkis.cli.common.entity.job.JobDescription; -import org.apache.linkis.cli.common.entity.job.JobSubType; -import org.apache.linkis.cli.common.entity.operator.JobOperator; -import org.apache.linkis.cli.common.entity.present.PresentWay; -import org.apache.linkis.cli.common.entity.result.ResultHandler; -import org.apache.linkis.cli.common.entity.validate.Validator; -import org.apache.linkis.cli.common.entity.var.VarAccess; -import org.apache.linkis.cli.common.exception.error.ErrorLevel; -import org.apache.linkis.cli.core.exception.LinkisClientExecutionException; -import org.apache.linkis.cli.core.exception.error.CommonErrMsg; -import org.apache.linkis.cli.core.interactor.execution.AsyncSubmission; -import org.apache.linkis.cli.core.interactor.execution.Help; -import org.apache.linkis.cli.core.interactor.execution.JobManagement; -import org.apache.linkis.cli.core.interactor.execution.SyncSubmission; -import org.apache.linkis.cli.core.interactor.job.JobBuilder; -import org.apache.linkis.cli.core.interactor.result.DefaultResultHandler; -import org.apache.linkis.cli.core.interactor.result.PresentResultHandler; -import org.apache.linkis.cli.core.present.DefaultStdOutPresenter; - -import org.apache.commons.lang3.StringUtils; - -import java.util.HashMap; -import java.util.Map; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public class ExecutionSuiteFactory { - private static Logger logger = LoggerFactory.getLogger(ExecutionSuiteFactory.class); - - public static ExecutionSuite getSuite( - CmdType cmdType, VarAccess stdVarAccess, VarAccess sysVarAccess) { - - ExecutionSuite suite; - Execution execution; - Map jobs = new HashMap<>(); - JobSubType subType; - JobBuilder jobBuilder; - Validator validator; - ResultHandler defaultHandler = new DefaultResultHandler(); - - /* - Prepare Builders and command-specific components - */ - if (cmdType == LinkisCmdType.UNIVERSAL) { - if (stdVarAccess.hasVar(AppKeys.LINKIS_CLIENT_KILL_OPT)) { - execution = new JobManagement(); - jobBuilder = new LinkisManageJobBuilder(); - PresentResultHandler handler = new PresentResultHandler(); - handler.setPresenter(new DefaultStdOutPresenter()); - handler.setModel(new LinkisJobKillModel()); - validator = new LinkisManageValidator(); - subType = LinkisManSubType.KILL; - suite = new ExecutionSuite(execution, jobs, handler, defaultHandler); - } else if (stdVarAccess.hasVar(AppKeys.LINKIS_CLIENT_STATUS_OPT)) { - execution = new JobManagement(); - jobBuilder = new LinkisManageJobBuilder(); - PresentResultHandler handler = new PresentResultHandler(); - handler.setPresenter(new DefaultStdOutPresenter()); - handler.setModel(new LinkisJobInfoModel()); - validator = new LinkisManageValidator(); - subType = LinkisManSubType.STATUS; - suite = new ExecutionSuite(execution, jobs, handler, defaultHandler); - } else if (stdVarAccess.hasVar(AppKeys.LINKIS_CLIENT_LOG_OPT)) { - execution = new JobManagement(); - jobBuilder = new LinkisManageJobBuilder().setLogListener(new LinkisLogPresenter()); - validator = new LinkisManageValidator(); - subType = LinkisManSubType.LOG; - suite = new ExecutionSuite(execution, jobs, null, defaultHandler); - } else if (stdVarAccess.hasVar(AppKeys.LINKIS_CLIENT_RESULT_OPT)) { - execution = new JobManagement(); - jobBuilder = new LinkisManageJobBuilder(); - PresentResultHandler handler = new PresentResultHandler(); - handler.setPresenter(new LinkisResultPresenter()); - handler.setModel(new LinkisResultModel()); - validator = new LinkisManageValidator(); - subType = LinkisManSubType.RESULT; - suite = new ExecutionSuite(execution, jobs, handler, defaultHandler); - } else if (stdVarAccess.hasVar(AppKeys.LINKIS_CLIENT_HELP_OPT)) { - execution = new Help(); - jobs.put( - "help", - new Job() { - @Override - public String getCid() { - return null; - } - - @Override - public CmdType getCmdType() { - return cmdType; - } - - @Override - public JobSubType getSubType() { - return null; - } - - @Override - public JobDescription getJobDesc() { - return null; - } - - @Override - public JobData getJobData() { - return null; - } - - @Override - public JobOperator getJobOperator() { - return null; - } - - @Override - public PresentWay getPresentWay() { - return null; - } - }); - return new ExecutionSuite(execution, jobs, null, defaultHandler); - } else { - Boolean asyncSubmission = - stdVarAccess.getVarOrDefault(Boolean.class, AppKeys.LINKIS_CLIENT_ASYNC_OPT, false); - if (asyncSubmission) { - execution = new AsyncSubmission(); - PresentResultHandler handler = new PresentResultHandler(); - handler.setPresenter(new DefaultStdOutPresenter()); - handler.setModel(new LinkisJobInfoModel()); - jobBuilder = new LinkisSubmitJobBuilder().setAsync(true); - subType = LinkisSubmitSubType.SUBMIT; - suite = new ExecutionSuite(execution, jobs, handler, defaultHandler); - validator = new LinkisSubmitValidator(); - } else { - execution = new SyncSubmission(); - subType = LinkisSubmitSubType.SUBMIT; - PresentResultHandler handler1 = new PresentResultHandler(); - handler1.setPresenter(new LinkisResultInfoPresenter()); - handler1.setModel(new LinkisResultInfoModel()); - PresentResultHandler handler2 = new PresentResultHandler(); - handler2.setPresenter(new LinkisResultPresenter()); - handler2.setModel(new LinkisResultModel()); - - String mode = - stdVarAccess.getVarOrDefault( - String.class, AppKeys.LINKIS_CLIENT_MODE_OPT, AppConstants.UJES_MODE); - if (StringUtils.equalsIgnoreCase(mode, AppConstants.ONCE_MODE)) { - jobBuilder = new LinkisOnceJobBuilder().setLogListener(new LinkisLogPresenter()); - ; - validator = new LinkisOnceSubmitValidator(); - } else { - jobBuilder = new LinkisSubmitJobBuilder().setLogListener(new LinkisLogPresenter()); - validator = new LinkisSubmitValidator(); - } - suite = new ExecutionSuite(execution, jobs, handler1, handler2, defaultHandler); - } - } - /* - build job - */ - Job job = - jobBuilder - .setCid(AppConstants.DUMMY_CID) // currently we don't need this - .setCmdType(cmdType) - .setJobSubType(subType) - .setStdVarAccess(stdVarAccess) - .setSysVarAccess(sysVarAccess) - .build(); - logger.info("==========JOB============\n" + Utils.GSON.toJson(job.getJobDesc())); - if (validator != null) { - validator.doValidation(job); - } - - jobs.put(job.getCid(), job); - - return suite; - } else { - throw new LinkisClientExecutionException( - "EXE0029", - ErrorLevel.ERROR, - CommonErrMsg.ExecutionInitErr, - "Command Type is not supported"); - } - } -} diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/utils/Utils.java b/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/utils/Utils.java deleted file mode 100644 index e70c32a37b..0000000000 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/utils/Utils.java +++ /dev/null @@ -1,52 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.cli.application.utils; - -import org.apache.commons.lang3.StringUtils; - -import com.google.gson.Gson; -import com.google.gson.GsonBuilder; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public class Utils { - - public static final Gson GSON = - new GsonBuilder().setPrettyPrinting().disableHtmlEscaping().create(); - private static final Logger logger = LoggerFactory.getLogger(Utils.class); - - public static boolean isValidExecId(String execId) { - boolean ret = false; - if (StringUtils.isNotBlank(execId)) { - ret = true; - } - return ret; - } - - public static String progressInPercentage(float progress) { - return String.valueOf(progress * 100) + "%"; - } - - public static void doSleepQuietly(Long sleepMills) { - try { - Thread.sleep(sleepMills); - } catch (Exception ignore) { - // ignored - } - } -} diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/test/java/org/apache/linkis/cli/application/interactor/command/template/ProcessInputUtil.java b/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/test/java/org/apache/linkis/cli/application/interactor/command/template/ProcessInputUtil.java deleted file mode 100644 index 33af32bb4b..0000000000 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/test/java/org/apache/linkis/cli/application/interactor/command/template/ProcessInputUtil.java +++ /dev/null @@ -1,127 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.cli.application.interactor.command.template; - -import org.apache.linkis.cli.application.constants.AppConstants; -import org.apache.linkis.cli.application.constants.AppKeys; -import org.apache.linkis.cli.application.data.ProcessedData; -import org.apache.linkis.cli.application.utils.Utils; -import org.apache.linkis.cli.common.entity.command.CmdTemplate; -import org.apache.linkis.cli.common.entity.command.Params; -import org.apache.linkis.cli.common.entity.properties.ClientProperties; -import org.apache.linkis.cli.common.entity.var.VarAccess; -import org.apache.linkis.cli.core.constants.CommonConstants; -import org.apache.linkis.cli.core.interactor.command.fitter.SingleTplFitter; -import org.apache.linkis.cli.core.interactor.command.parser.Parser; -import org.apache.linkis.cli.core.interactor.command.parser.SingleCmdParser; -import org.apache.linkis.cli.core.interactor.command.parser.result.ParseResult; -import org.apache.linkis.cli.core.interactor.properties.PropertiesLoader; -import org.apache.linkis.cli.core.interactor.properties.PropsFilesScanner; -import org.apache.linkis.cli.core.interactor.properties.StdPropsLoader; -import org.apache.linkis.cli.core.interactor.properties.reader.PropertiesReader; -import org.apache.linkis.cli.core.interactor.properties.reader.PropsFileReader; -import org.apache.linkis.cli.core.interactor.properties.reader.SysEnvReader; -import org.apache.linkis.cli.core.interactor.properties.reader.SysPropsReader; -import org.apache.linkis.cli.core.interactor.validate.ParsedTplValidator; -import org.apache.linkis.cli.core.interactor.var.StdVarAccess; -import org.apache.linkis.cli.core.interactor.var.SysVarAccess; -import org.apache.linkis.cli.core.utils.LogUtils; - -import org.apache.commons.lang3.StringUtils; - -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public class ProcessInputUtil { - private static Logger logger = LoggerFactory.getLogger(ProcessInputUtil.class); - - public static ProcessedData generateProcessedData(String[] cmdStr, CmdTemplate template) { - /* - user input - */ - Parser parser = - new SingleCmdParser() - .setMapper(null) - .setTemplate(template) - .setFitter(new SingleTplFitter()); - - ParseResult result = parser.parse(cmdStr); - - ParsedTplValidator parsedTplValidator = new ParsedTplValidator(); - parsedTplValidator.doValidation(result.getParsedTemplate()); - - Params params = result.getParams(); - logger.debug("==========params============\n" + Utils.GSON.toJson(params)); - - Map propertiesMap = new HashMap<>(); - /* - default config, -Dconf.root & -Dconf.file specifies config path - */ - System.setProperty("conf.root", "src/test/resources/conf/"); - System.setProperty("conf.file", "linkis-cli.properties"); - String configPath = System.getProperty("conf.root"); - String defaultConfFileName = System.getProperty("conf.file"); - List readersList = - new PropsFilesScanner().getPropsReaders(configPath); // +1 user config - /* - user defined config - */ - String userConfPath = null; - if (params.containsParam(AppKeys.LINKIS_CLIENT_USER_CONFIG)) { - userConfPath = - (String) params.getParamItemMap().get(AppKeys.LINKIS_CLIENT_USER_CONFIG).getValue(); - } - if (StringUtils.isNotBlank(userConfPath)) { - PropertiesReader reader = - new PropsFileReader() - .setPropsId(AppKeys.LINKIS_CLIENT_USER_CONFIG) - .setPropsPath(userConfPath); - readersList.add(reader); - } else { - LogUtils.getInformationLogger() - .info("User does not provide usr-configuration file. Will use default config"); - } - readersList.add(new SysPropsReader()); - readersList.add(new SysEnvReader()); - PropertiesLoader loader = - new StdPropsLoader() - .addPropertiesReaders(readersList.toArray(new PropertiesReader[readersList.size()])); - ClientProperties[] loaderResult = loader.loadProperties(); - for (ClientProperties properties : loaderResult) { - propertiesMap.put(properties.getPropsId(), properties); - } - - VarAccess stdVarAccess = - new StdVarAccess() - .setCmdParams(params) - .setUserConf(propertiesMap.get(AppKeys.LINKIS_CLIENT_USER_CONFIG)) - .setDefaultConf(propertiesMap.get(AppConstants.DEFAULT_CONFIG_NAME)) - .init(); - - VarAccess sysVarAccess = - new SysVarAccess() - .setSysProp(propertiesMap.get(CommonConstants.SYSTEM_PROPERTIES_IDENTIFIER)) - .setSysEnv(propertiesMap.get(CommonConstants.SYSTEM_ENV_IDENTIFIER)); - - return new ProcessedData(null, params.getCmdType(), stdVarAccess, sysVarAccess); - } -} diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/test/java/org/apache/linkis/cli/application/interactor/execution/LinkisExecutorTest.java b/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/test/java/org/apache/linkis/cli/application/interactor/execution/LinkisExecutorTest.java deleted file mode 100644 index b2571049d3..0000000000 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/test/java/org/apache/linkis/cli/application/interactor/execution/LinkisExecutorTest.java +++ /dev/null @@ -1,113 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.cli.application.interactor.execution; - -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; - -public class LinkisExecutorTest { - - @BeforeEach - public void before() {} - - @AfterEach - public void after() {} - - /** Method: setDriver(LinkisClientDriver driver) */ - @Test - public void testSetDriver() { - // TODO: Test goes here... - } - - /** Method: setDriverTransformer(DriverTransformer driverTransformer) */ - @Test - public void testSetDriverTransformer() { - // TODO: Test goes here... - } - - /** Method: getJobID(JobSubmitExec execData) */ - @Test - public void testGetJobID() { - // TODO: Test goes here... - } - - /** Method: submit(Job job) */ - @Test - public void testDoAsyncSubmit() { - // TODO: Test goes here... - } - - /** Method: updateJobStatus(JobSubmitExec execData) */ - @Test - public void testDoUpdateProgress() { - // TODO: Test goes here... - } - - /** Method: checkSubmit(JobSubmitExec execData) */ - @Test - public void testDoCheckSubmit() { - // TODO: Test goes here... - } - - /** Method: doGetFinalResult(JobSubmitExec execData) */ - @Test - public void testDoGetFinalResult() { - // TODO: Test goes here... - } - - /** Method: checkInit() */ - @Test - public void testCheckInit() { - // TODO: Test goes here... - } - - /** Method: doKill(Job job) */ - @Test - public void testDoKill() { - // TODO: Test goes here... - } - - /** Method: doQuery(Job job) */ - @Test - public void testDoQuery() { - // TODO: Test goes here... - } - - /** Method: doTransform(Job job) */ - @Test - public void testDoTransform() { - // TODO: Test goes here... - } - - /** Method: updateExecDataByDwsResult(JobSubmitExec execData, DWSResult result) */ - @Test - public void testUpdateExecDataByDwsResult() { - // TODO: Test goes here... - /* - try { - Method method = LinkisSubmitExecutor.getClass().getMethod("updateExecDataByDwsResult", JobSubmitExec.class, DWSResult.class); - method.setAccessible(true); - method.invoke(, ); - } catch(NoSuchMethodException e) { - } catch(IllegalAccessException e) { - } catch(InvocationTargetException e) { - } - */ - } -} diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/test/java/org/apache/linkis/cli/application/interactor/job/LinkisJobBuilderTest.java b/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/test/java/org/apache/linkis/cli/application/interactor/job/LinkisJobBuilderTest.java deleted file mode 100644 index 99836e8ec6..0000000000 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/test/java/org/apache/linkis/cli/application/interactor/job/LinkisJobBuilderTest.java +++ /dev/null @@ -1,106 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.cli.application.interactor.job; - -import org.apache.linkis.cli.application.data.ProcessedData; -import org.apache.linkis.cli.application.interactor.command.template.ProcessInputUtil; -import org.apache.linkis.cli.application.interactor.command.template.TestSparkCmdTemplate; -import org.apache.linkis.cli.core.interactor.job.JobBuilder; - -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; - -public class LinkisJobBuilderTest { - JobBuilder builder; - - @BeforeEach - public void before() { - String[] cmdStr = - new String[] { - "-u", - "hadoop", - "-pwd", - "1234", - "-c", - "src/test/resources/conf/user.properties", - "--cmd", - "show tables", - "--split", - "\',\'", - "--queue", - "q05", - "--name", - "testApp", - // "--hiveconf", "/path/...", - // "--num-executors", "4", - "--executor-cores", - "4", - "--executor-memory", - "4G", - "--shuffle-partitions", - "200", - "-confMap", - "kk=vv,kkk=vvv,spark.executor.instances=8" - }; - ProcessedData data = ProcessInputUtil.generateProcessedData(cmdStr, new TestSparkCmdTemplate()); - // builder = new LinkisJobBuilder() - // .setIdentifier(data.getIdentifier()) - // .setCmdType(data.getCmdType()) - // .setExecutionType(data.getExecutionType()) - // .setSubExecutionType(data.getSubExecutionType()) - // .setStdVarAccess(data.getStdVarAccess()) - // .setSysVarAccess(data.getSysVarAccess()); - } - - @AfterEach - public void after() {} - - /** Method: setIdentifier(String identifier) */ - @Test - public void testSetIdentifier() { - // TODO: Test goes here... - } - - /** Method: build() */ - @Test - public void testBuild() { - // TODO: Test goes here... - // LinkisJob job = builder.build(); - // System.out.println(CommonUtils.GSON.toJson(job)); - // assertEquals(job.getParamConfMap().get("spark.executor.instances"), "8"); - } - - /** Method: setStdVarAccess(VarAccess varAccess) */ - @Test - public void testSetStdVarAccess() { - // TODO: Test goes here... - } - - /** Method: setSysVarAccess(VarAccess varAccess) */ - @Test - public void testSetSysVarAccess() { - // TODO: Test goes here... - } - - /** Method: getTargetNewInstance() */ - @Test - public void testGetTargetNewInstance() { - // TODO: Test goes here... - } -} diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/test/java/org/apache/linkis/cli/application/interactor/job/subtype/LinkisSubmitSubTypeTest.java b/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/test/java/org/apache/linkis/cli/application/interactor/job/subtype/LinkisSubmitSubTypeTest.java deleted file mode 100644 index f05d59700e..0000000000 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/test/java/org/apache/linkis/cli/application/interactor/job/subtype/LinkisSubmitSubTypeTest.java +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.cli.application.interactor.job.subtype; - -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.DisplayName; -import org.junit.jupiter.api.Test; - -public class LinkisSubmitSubTypeTest { - - @Test - @DisplayName("enumTest") - public void enumTest() { - - String submitName = LinkisSubmitSubType.SUBMIT.getName(); - Assertions.assertEquals("submit", submitName); - } -} diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-common/pom.xml b/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-common/pom.xml deleted file mode 100644 index 4cf0817130..0000000000 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-common/pom.xml +++ /dev/null @@ -1,29 +0,0 @@ - - - - 4.0.0 - - - org.apache.linkis - linkis-cli - ${revision} - - linkis-cli-common - jar - - diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-common/src/main/java/org/apache/linkis/cli/common/entity/execution/Execution.java b/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-common/src/main/java/org/apache/linkis/cli/common/entity/execution/Execution.java deleted file mode 100644 index 23e99601eb..0000000000 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-common/src/main/java/org/apache/linkis/cli/common/entity/execution/Execution.java +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.cli.common.entity.execution; - -import org.apache.linkis.cli.common.entity.job.Job; -import org.apache.linkis.cli.common.entity.result.ExecutionResult; - -import java.util.Map; - -public interface Execution { - /* - start execution - */ - ExecutionResult execute(Map jobs); - - /* - terminate execution(often used in shutdown hook) - */ - boolean terminate(Map jobs); -} diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-common/src/main/java/org/apache/linkis/cli/common/entity/job/Job.java b/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-common/src/main/java/org/apache/linkis/cli/common/entity/job/Job.java deleted file mode 100644 index 5339db3c09..0000000000 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-common/src/main/java/org/apache/linkis/cli/common/entity/job/Job.java +++ /dev/null @@ -1,55 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.cli.common.entity.job; - -import org.apache.linkis.cli.common.entity.command.CmdTemplate; -import org.apache.linkis.cli.common.entity.command.CmdType; -import org.apache.linkis.cli.common.entity.operator.JobOperator; -import org.apache.linkis.cli.common.entity.present.PresentWay; - -public interface Job { - /** Linkis-cli specified id, not server-side returned job-id/task-id */ - String getCid(); - - /** - * Command Type for this Job, should be able to use this to find out corresponding {@link - * CmdTemplate} - */ - CmdType getCmdType(); - - /** specifies which kind of sub-execution: e.g. jobManagement: status/list/log/kill; */ - JobSubType getSubType(); - - /** - * input-param/config will be stored in JobDescription information contained by this - * data-structure should be passed to server - */ - JobDescription getJobDesc(); - - /** - * data generated during execution(e.g. job status, job id, log, result etc.) is stored here - * information contained by this data-structure can be further passed to server - */ - JobData getJobData(); - - /** operates lower level components(usually encapsulates a client) */ - JobOperator getJobOperator(); - - /** decide how result should be presented */ - PresentWay getPresentWay(); -} diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-common/src/main/java/org/apache/linkis/cli/common/entity/job/JobData.java b/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-common/src/main/java/org/apache/linkis/cli/common/entity/job/JobData.java deleted file mode 100644 index 10d55e6541..0000000000 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-common/src/main/java/org/apache/linkis/cli/common/entity/job/JobData.java +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.cli.common.entity.job; - -public interface JobData { - JobStatus getJobStatus(); - - String getJobID(); - - String getUser(); - - String getMessage(); - - void setMessage(String message); - - Exception getException(); - - void setException(Exception e); -} diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-common/src/main/java/org/apache/linkis/cli/common/entity/job/JobDescription.java b/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-common/src/main/java/org/apache/linkis/cli/common/entity/job/JobDescription.java deleted file mode 100644 index f6fa3e3217..0000000000 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-common/src/main/java/org/apache/linkis/cli/common/entity/job/JobDescription.java +++ /dev/null @@ -1,21 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.cli.common.entity.job; - -/** Should store all input argument and configurations */ -public interface JobDescription {} diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-common/src/main/java/org/apache/linkis/cli/common/entity/operator/JobOperator.java b/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-common/src/main/java/org/apache/linkis/cli/common/entity/operator/JobOperator.java deleted file mode 100644 index e4bef68012..0000000000 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-common/src/main/java/org/apache/linkis/cli/common/entity/operator/JobOperator.java +++ /dev/null @@ -1,20 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.cli.common.entity.operator; - -public interface JobOperator {} diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-common/src/main/java/org/apache/linkis/cli/common/entity/present/PresentWay.java b/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-common/src/main/java/org/apache/linkis/cli/common/entity/present/PresentWay.java deleted file mode 100644 index a91dd3d475..0000000000 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-common/src/main/java/org/apache/linkis/cli/common/entity/present/PresentWay.java +++ /dev/null @@ -1,20 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.cli.common.entity.present; - -public interface PresentWay {} diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-common/src/main/java/org/apache/linkis/cli/common/entity/result/ExecutionResult.java b/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-common/src/main/java/org/apache/linkis/cli/common/entity/result/ExecutionResult.java deleted file mode 100644 index 3d5cdef7e3..0000000000 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-common/src/main/java/org/apache/linkis/cli/common/entity/result/ExecutionResult.java +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.cli.common.entity.result; - -import org.apache.linkis.cli.common.entity.job.Job; - -import java.util.Map; - -public interface ExecutionResult { - Map getJobs(); - - ExecutionStatus getExecutionStatus(); - - void setExecutionStatus(ExecutionStatus executionStatus); - - Exception getException(); // TODO: put exception during execution in here and do not interrupt - // execution - - void setException( - Exception exception); // TODO: put exception during execution in here and do not interrupt - // execution -} diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-common/src/main/java/org/apache/linkis/cli/common/entity/result/ExecutionStatus.java b/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-common/src/main/java/org/apache/linkis/cli/common/entity/result/ExecutionStatus.java deleted file mode 100644 index 744ce3433a..0000000000 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-common/src/main/java/org/apache/linkis/cli/common/entity/result/ExecutionStatus.java +++ /dev/null @@ -1,20 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.cli.common.entity.result; - -public interface ExecutionStatus {} diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-common/src/main/java/org/apache/linkis/cli/common/entity/result/ResultHandler.java b/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-common/src/main/java/org/apache/linkis/cli/common/entity/result/ResultHandler.java deleted file mode 100644 index cf10cb2bf4..0000000000 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-common/src/main/java/org/apache/linkis/cli/common/entity/result/ResultHandler.java +++ /dev/null @@ -1,22 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.cli.common.entity.result; - -public interface ResultHandler { - void process(ExecutionResult executionResult); -} diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-common/src/main/java/org/apache/linkis/cli/common/exception/handler/ExceptionHandler.java b/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-common/src/main/java/org/apache/linkis/cli/common/exception/handler/ExceptionHandler.java deleted file mode 100644 index cb5a756ac3..0000000000 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-common/src/main/java/org/apache/linkis/cli/common/exception/handler/ExceptionHandler.java +++ /dev/null @@ -1,22 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.cli.common.exception.handler; - -public interface ExceptionHandler { - void handle(Exception e); -} diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-common/src/test/java/org/apache/linkis/cli/common/exception/error/ErrorLevelTest.java b/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-common/src/test/java/org/apache/linkis/cli/common/exception/error/ErrorLevelTest.java deleted file mode 100644 index e6d14f71ea..0000000000 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-common/src/test/java/org/apache/linkis/cli/common/exception/error/ErrorLevelTest.java +++ /dev/null @@ -1,42 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.cli.common.exception.error; - -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.DisplayName; -import org.junit.jupiter.api.Test; - -public class ErrorLevelTest { - - @Test - @DisplayName("enumTest") - public void enumTest() { - - int infoLevel = ErrorLevel.INFO.getLevel(); - int warnLevel = ErrorLevel.WARN.getLevel(); - int errorLevel = ErrorLevel.ERROR.getLevel(); - int fatalLevel = ErrorLevel.FATAL.getLevel(); - int retryLevel = ErrorLevel.RETRY.getLevel(); - - Assertions.assertTrue(0 == infoLevel); - Assertions.assertTrue(1 == warnLevel); - Assertions.assertTrue(2 == errorLevel); - Assertions.assertTrue(3 == fatalLevel); - Assertions.assertTrue(4 == retryLevel); - } -} diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/pom.xml b/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/pom.xml deleted file mode 100644 index 4d453144fb..0000000000 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/pom.xml +++ /dev/null @@ -1,43 +0,0 @@ - - - - 4.0.0 - - - org.apache.linkis - linkis-cli - ${revision} - - linkis-cli-core - jar - - - - org.apache.linkis - linkis-cli-common - ${project.version} - - - - org.reflections - reflections - - - - - diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/builder/AbstractBuilder.java b/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/builder/AbstractBuilder.java deleted file mode 100644 index 8063662cb5..0000000000 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/builder/AbstractBuilder.java +++ /dev/null @@ -1,54 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.cli.core.builder; - -public abstract class AbstractBuilder implements Builder { - protected T targetObj; - - public AbstractBuilder() { - reset(); - } - - protected void reset() { - targetObj = getTargetNewInstance(); - } - - @Override - public T build() { - T ret = targetObj; - reset(); - return ret; - } - - protected abstract T getTargetNewInstance(); - - // protected T getTargetNewInstance() { - // try { - // ParameterizedType pt = (ParameterizedType) this.getClass().getGenericSuperclass(); - // Class clazz = (Class) pt.getActualTypeArguments()[0]; - // targetObj = clazz.newInstance(); - // } catch (InstantiationException e) { - // throw new BuilderException("BLD0001", ErrorLevel.ERROR, - // CommonErrMsg.BuilderBuildErr, "Cannot generate new instance.", e); - // } catch (IllegalAccessException ie) { - // throw new BuilderException("BLD0001", ErrorLevel.ERROR, - // CommonErrMsg.BuilderBuildErr, "Cannot generate new instance.", ie); - // } - // return targetObj; - // } -} diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/builder/BuildableByVarAccess.java b/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/builder/BuildableByVarAccess.java deleted file mode 100644 index ca6454e923..0000000000 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/builder/BuildableByVarAccess.java +++ /dev/null @@ -1,56 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.cli.core.builder; - -import org.apache.linkis.cli.common.entity.var.VarAccess; -import org.apache.linkis.cli.common.exception.error.ErrorLevel; -import org.apache.linkis.cli.core.exception.BuilderException; -import org.apache.linkis.cli.core.exception.error.CommonErrMsg; - -import java.lang.reflect.ParameterizedType; - -public abstract class BuildableByVarAccess extends AbstractBuilder { - protected VarAccess stdVarAccess; - protected VarAccess sysVarAccess; - - protected void checkInit() { - if (stdVarAccess == null || sysVarAccess == null) { - ParameterizedType pt = (ParameterizedType) this.getClass().getGenericSuperclass(); - Class clazz = (Class) pt.getActualTypeArguments()[0]; - throw new BuilderException( - "BLD0003", - ErrorLevel.ERROR, - CommonErrMsg.BuilderInitErr, - "Cannot init builder: " - + clazz.getCanonicalName() - + "Cause: stdVarAccess or sysVarAccess is null"); - } - stdVarAccess.checkInit(); - sysVarAccess.checkInit(); - } - - public BuildableByVarAccess setStdVarAccess(VarAccess varAccess) { - this.stdVarAccess = varAccess; - return this; - } - - public BuildableByVarAccess setSysVarAccess(VarAccess varAccess) { - this.sysVarAccess = varAccess; - return this; - } -} diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/builder/Builder.java b/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/builder/Builder.java deleted file mode 100644 index e66ba2ee55..0000000000 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/builder/Builder.java +++ /dev/null @@ -1,22 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.cli.core.builder; - -public interface Builder { - T build(); -} diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/constants/CommonConstants.java b/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/constants/CommonConstants.java deleted file mode 100644 index 1362539158..0000000000 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/constants/CommonConstants.java +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.cli.core.constants; - -public class CommonConstants { - - public static final Long JOB_QUERY_SLEEP_MILLS = 2000l; - public static final Integer REQUEST_MAX_RETRY_TIME = 3; - - public static final String UNIVERSAL_SUBCMD = "linkis-cli"; - public static final String UNIVERSAL_SUBCMD_DESC = - "command for all types of jobs supported by Linkis"; - - public static final String SUCCESS_INDICATOR = "############Execute Success!!!########"; - public static final String FAILURE_INDICATOR = "############Execute Error!!!########"; - public static final String ARRAY_SEQ = "@#@"; - public static final String ARRAY_SEQ_REGEX = "(?=([^\"]*\"[^\"]*\")*[^\"]*$)"; - public static final int MAX_NUM_OF_COMMAND_ARGUEMENTS = 10; - public static final String CONFIG_DIR = "config.path"; - public static final String[] CONFIG_EXTENSION = {"properties"}; - public static final String SYSTEM_PROPERTIES_IDENTIFIER = "SYS_PROP"; - public static final String SYSTEM_ENV_IDENTIFIER = "SYS_ENV"; -} diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/constants/CommonKeys.java b/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/constants/CommonKeys.java deleted file mode 100644 index 39eb2f9418..0000000000 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/constants/CommonKeys.java +++ /dev/null @@ -1,20 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.cli.core.constants; - -public class CommonKeys {} diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/exception/handler/CommandExceptionHandler.java b/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/exception/handler/CommandExceptionHandler.java deleted file mode 100644 index 25d7a04d20..0000000000 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/exception/handler/CommandExceptionHandler.java +++ /dev/null @@ -1,46 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.cli.core.exception.handler; - -import org.apache.linkis.cli.common.entity.command.CmdTemplate; -import org.apache.linkis.cli.common.exception.handler.ExceptionHandler; -import org.apache.linkis.cli.core.exception.CommandException; -import org.apache.linkis.cli.core.interactor.command.CmdTemplateFactory; -import org.apache.linkis.cli.core.present.HelpInfoPresenter; -import org.apache.linkis.cli.core.present.model.HelpInfoModel; - -public class CommandExceptionHandler implements ExceptionHandler { - @Override - public void handle(Exception e) { - if (e instanceof CommandException) { - if (((CommandException) e).requireHelp()) { - - CmdTemplate template = - CmdTemplateFactory.getTemplateOri(((CommandException) e).getCmdType()); - - if (template != null) { - HelpInfoModel model = new HelpInfoModel(); - model.buildModel(template); - - new HelpInfoPresenter().present(model, null); - } - } - } - new DefaultExceptionHandler().handle(e); - } -} diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/exception/handler/DefaultExceptionHandler.java b/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/exception/handler/DefaultExceptionHandler.java deleted file mode 100644 index c83e711513..0000000000 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/exception/handler/DefaultExceptionHandler.java +++ /dev/null @@ -1,83 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.cli.core.exception.handler; - -import org.apache.linkis.cli.common.exception.LinkisClientRuntimeException; -import org.apache.linkis.cli.common.exception.handler.ExceptionHandler; -import org.apache.linkis.cli.core.utils.LogUtils; - -import org.apache.commons.lang3.StringUtils; -import org.apache.commons.lang3.exception.ExceptionUtils; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public class DefaultExceptionHandler implements ExceptionHandler { - private static Logger logger = LoggerFactory.getLogger(DefaultExceptionHandler.class); - - @Override - public void handle(Exception exception) { - if (exception instanceof LinkisClientRuntimeException) { - LinkisClientRuntimeException e = (LinkisClientRuntimeException) exception; - switch (e.getLevel()) { - case INFO: - logger.info(e.getMessage(), e); - LogUtils.getInformationLogger().info(e.getMessage()); - break; - case WARN: - logger.warn(e.getMessage(), e); - LogUtils.getInformationLogger().warn(getErrInfoWithoutStacktrace(e)); - break; - case ERROR: - logger.error(e.getMessage(), e); - LogUtils.getInformationLogger().error(getErrInfoWithoutStacktrace(e)); - break; - case FATAL: - String msg = StringUtils.substringAfter(e.getMessage(), "[ERROR]"); - logger.error(msg, e); - LogUtils.getInformationLogger().error("[FATAL]" + msg, e); - System.exit(-1); - break; - } - - } else { - logger.error(exception.getMessage(), exception); - LogUtils.getInformationLogger().error(exception.getMessage(), exception); - } - } - - private String getErrInfoWithoutStacktrace(Exception e) { - if (e == null) { - return ""; - } - StringBuilder sb = new StringBuilder(); - if (e instanceof NullPointerException) { - sb.append(ExceptionUtils.getStackTrace(e)); - } else { - sb.append(e.getMessage()); - } - if (e.getCause() != null) { - sb.append(System.lineSeparator()) - .append("Caused by: ") - .append((e.getCause().getClass().getCanonicalName())) - .append(": ") - .append(e.getCause().getMessage()); - } - return sb.toString(); - } -} diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/execution/AsyncSubmission.java b/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/execution/AsyncSubmission.java deleted file mode 100644 index 765c5969d4..0000000000 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/execution/AsyncSubmission.java +++ /dev/null @@ -1,109 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.cli.core.interactor.execution; - -import org.apache.linkis.cli.common.entity.execution.Execution; -import org.apache.linkis.cli.common.entity.job.Job; -import org.apache.linkis.cli.common.entity.result.ExecutionResult; -import org.apache.linkis.cli.common.entity.result.ExecutionStatus; -import org.apache.linkis.cli.common.exception.error.ErrorLevel; -import org.apache.linkis.cli.core.constants.CommonConstants; -import org.apache.linkis.cli.core.exception.LinkisClientExecutionException; -import org.apache.linkis.cli.core.exception.error.CommonErrMsg; -import org.apache.linkis.cli.core.interactor.job.AsyncBackendJob; -import org.apache.linkis.cli.core.interactor.result.ExecutionResultImpl; -import org.apache.linkis.cli.core.interactor.result.ExecutionStatusEnum; -import org.apache.linkis.cli.core.utils.CommonUtils; - -import java.util.Map; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * Execute job asynchronously TODO: put exception during execution in ExecutionResult and do not - * interrupt execution - */ -public class AsyncSubmission implements Execution { - private static final Logger logger = LoggerFactory.getLogger(AsyncSubmission.class); - - @Override - public ExecutionResult execute(Map jobs) { - ExecutionStatus executionStatus; - Exception exception = null; // TODO - - if (jobs == null || jobs.size() == 0) { - throw new LinkisClientExecutionException( - "EXE0001", - ErrorLevel.ERROR, - CommonErrMsg.ExecutionInitErr, - "Null or empty Jobs is submitted to current execution"); - } - - if (jobs.size() > 1) { - throw new LinkisClientExecutionException( - "EXE0001", - ErrorLevel.ERROR, - CommonErrMsg.ExecutionInitErr, - "Multiple Jobs is not Supported by current execution"); - } - - Job job = jobs.get(jobs.keySet().toArray(new String[jobs.size()])[0]); - - if (!(job instanceof AsyncBackendJob)) { - throw new LinkisClientExecutionException( - "EXE0001", - ErrorLevel.ERROR, - CommonErrMsg.ExecutionInitErr, - "Backend for \"" + job.getClass().getCanonicalName() + "\" does not support async"); - } - - if (job.getSubType() == null) { - throw new LinkisClientExecutionException( - "EXE0001", - ErrorLevel.ERROR, - CommonErrMsg.ExecutionInitErr, - "SubExecType should not be null"); - } - - try { - ((AsyncBackendJob) job).submit(); - CommonUtils.doSleepQuietly(CommonConstants.JOB_QUERY_SLEEP_MILLS); - ((AsyncBackendJob) job).updateJobStatus(); - if (job.getJobData().getJobStatus().isJobSubmitted()) { - executionStatus = ExecutionStatusEnum.SUCCEED; - } else { - executionStatus = ExecutionStatusEnum.FAILED; - if (job.getJobData().getException() != null) { - exception = job.getJobData().getException(); - } - } - } catch (Exception e) { - exception = e; - executionStatus = ExecutionStatusEnum.FAILED; - logger.warn("Failed to submit job.", e); - } - - return new ExecutionResultImpl(jobs, executionStatus, exception); - } - - @Override - public boolean terminate(Map jobs) { - return true; - } -} diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/execution/Help.java b/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/execution/Help.java deleted file mode 100644 index b2d34746c6..0000000000 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/execution/Help.java +++ /dev/null @@ -1,59 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.cli.core.interactor.execution; - -import org.apache.linkis.cli.common.entity.execution.Execution; -import org.apache.linkis.cli.common.entity.job.Job; -import org.apache.linkis.cli.common.entity.result.ExecutionResult; -import org.apache.linkis.cli.common.exception.error.ErrorLevel; -import org.apache.linkis.cli.core.exception.LinkisClientExecutionException; -import org.apache.linkis.cli.core.exception.error.CommonErrMsg; -import org.apache.linkis.cli.core.interactor.command.CmdTemplateFactory; -import org.apache.linkis.cli.core.interactor.result.ExecutionResultImpl; -import org.apache.linkis.cli.core.interactor.result.ExecutionStatusEnum; -import org.apache.linkis.cli.core.present.HelpInfoPresenter; -import org.apache.linkis.cli.core.present.model.HelpInfoModel; - -import java.util.Map; - -public class Help implements Execution { - @Override - public ExecutionResult execute(Map jobs) { - - if (jobs.size() > 1) { - throw new LinkisClientExecutionException( - "EXE0001", - ErrorLevel.ERROR, - CommonErrMsg.ExecutionInitErr, - "Multiple Jobs is not Supported by current execution"); - } - - HelpInfoModel model = new HelpInfoModel(); - Job job = jobs.values().toArray(new Job[jobs.size()])[0]; - - model.buildModel(CmdTemplateFactory.getTemplateOri(job.getCmdType())); - - new HelpInfoPresenter().present(model, null); - return new ExecutionResultImpl(null, ExecutionStatusEnum.SUCCEED, null); - } - - @Override - public boolean terminate(Map jobs) { - return true; - } -} diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/execution/JobManagement.java b/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/execution/JobManagement.java deleted file mode 100644 index c4741bdf68..0000000000 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/execution/JobManagement.java +++ /dev/null @@ -1,96 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.cli.core.interactor.execution; - -import org.apache.linkis.cli.common.entity.execution.Execution; -import org.apache.linkis.cli.common.entity.job.Job; -import org.apache.linkis.cli.common.entity.result.ExecutionResult; -import org.apache.linkis.cli.common.entity.result.ExecutionStatus; -import org.apache.linkis.cli.common.exception.error.ErrorLevel; -import org.apache.linkis.cli.core.exception.LinkisClientExecutionException; -import org.apache.linkis.cli.core.exception.error.CommonErrMsg; -import org.apache.linkis.cli.core.interactor.job.ManagableBackendJob; -import org.apache.linkis.cli.core.interactor.result.ExecutionResultImpl; -import org.apache.linkis.cli.core.interactor.result.ExecutionStatusEnum; - -import java.util.Map; - -public class JobManagement implements Execution { - @Override - public ExecutionResult execute(Map jobs) { - ExecutionStatus executionStatus; - Exception exception = null; // TODO - - if (jobs == null || jobs.size() == 0) { - throw new LinkisClientExecutionException( - "EXE0001", - ErrorLevel.ERROR, - CommonErrMsg.ExecutionInitErr, - "Null or empty Jobs is submitted to current execution"); - } - - if (jobs.size() > 1) { - throw new LinkisClientExecutionException( - "EXE0001", - ErrorLevel.ERROR, - CommonErrMsg.ExecutionInitErr, - "Multiple Jobs is not Supported by current execution"); - } - - Job job = jobs.get(jobs.keySet().toArray(new String[jobs.size()])[0]); - - if (!(job instanceof ManagableBackendJob)) { - throw new LinkisClientExecutionException( - "EXE0001", - ErrorLevel.ERROR, - CommonErrMsg.ExecutionInitErr, - "Backend for \"" + job.getClass().getCanonicalName() + "\" is not manageable"); - } - - if (job.getSubType() == null) { - throw new LinkisClientExecutionException( - "EXE0001", - ErrorLevel.ERROR, - CommonErrMsg.ExecutionInitErr, - "SubExecType should not be null"); - } - - try { - ((ManagableBackendJob) job).doManage(); - if (((ManagableBackendJob) job).isSuccess()) { - executionStatus = ExecutionStatusEnum.SUCCEED; - } else { - executionStatus = ExecutionStatusEnum.FAILED; - if (job.getJobData() != null && job.getJobData().getException() != null) { - exception = job.getJobData().getException(); - } - } - - } catch (Exception e) { - exception = e; - executionStatus = ExecutionStatusEnum.FAILED; - } - - return new ExecutionResultImpl(jobs, executionStatus, exception); - } - - @Override - public boolean terminate(Map jobs) { - return true; - } -} diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/execution/SyncSubmission.java b/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/execution/SyncSubmission.java deleted file mode 100644 index c143e22357..0000000000 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/execution/SyncSubmission.java +++ /dev/null @@ -1,195 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.cli.core.interactor.execution; - -import org.apache.linkis.cli.common.entity.execution.Execution; -import org.apache.linkis.cli.common.entity.job.Job; -import org.apache.linkis.cli.common.entity.result.ExecutionResult; -import org.apache.linkis.cli.common.entity.result.ExecutionStatus; -import org.apache.linkis.cli.common.exception.error.ErrorLevel; -import org.apache.linkis.cli.core.constants.CommonConstants; -import org.apache.linkis.cli.core.exception.LinkisClientExecutionException; -import org.apache.linkis.cli.core.exception.error.CommonErrMsg; -import org.apache.linkis.cli.core.interactor.job.*; -import org.apache.linkis.cli.core.interactor.result.ExecutionResultImpl; -import org.apache.linkis.cli.core.interactor.result.ExecutionStatusEnum; -import org.apache.linkis.cli.core.utils.CommonUtils; -import org.apache.linkis.cli.core.utils.LogUtils; - -import org.apache.commons.lang3.exception.ExceptionUtils; - -import java.util.Map; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * Execute job synchronously. i.e. Client submit job, and wait til job finish, and get result, no - * matter what server behaves. - */ -public class SyncSubmission implements Execution { - private static final Logger logger = LoggerFactory.getLogger(SyncSubmission.class); - - @Override - public ExecutionResult execute(Map jobs) { - - ExecutionStatus executionStatus; - Exception exception = null; // TODO - - if (jobs == null || jobs.size() == 0) { - throw new LinkisClientExecutionException( - "EXE0001", - ErrorLevel.ERROR, - CommonErrMsg.ExecutionErr, - "Null or empty Jobs is submitted to current execution"); - } - - if (jobs.size() > 1) { - throw new LinkisClientExecutionException( - "EXE0001", - ErrorLevel.ERROR, - CommonErrMsg.ExecutionErr, - "Multiple Jobs is not Supported by current execution"); - } - - Job job = jobs.get(jobs.keySet().toArray(new String[jobs.size()])[0]); - - if (job instanceof SyncBackendJob) { - try { - ((SyncBackendJob) job).submitAndGetResult(); - } catch (Exception e) { - exception = e; - } - } else if (job instanceof AsyncBackendJob) { - try { - ExecWithAsyncBackend(job); - } catch (Exception e) { - exception = e; - // TODO: throw or fail - } - } else { - throw new LinkisClientExecutionException( - "EXE0002", - ErrorLevel.ERROR, - CommonErrMsg.ExecutionErr, - "Executor Type: \"" + job.getClass().getCanonicalName() + "\" is not Supported"); - } - - if (job.getJobData() != null - && job.getJobData().getJobStatus() != null - && job.getJobData().getJobStatus().isJobSuccess()) { - executionStatus = ExecutionStatusEnum.SUCCEED; - } else if (job.getJobData().getJobStatus() == null - || !job.getJobData().getJobStatus().isJobFinishedState()) { - executionStatus = ExecutionStatusEnum.UNDEFINED; - if (job.getJobData().getException() != null) { - exception = job.getJobData().getException(); - } - } else { - executionStatus = ExecutionStatusEnum.FAILED; - if (job.getJobData().getException() != null) { - exception = job.getJobData().getException(); - } - } - - return new ExecutionResultImpl(jobs, executionStatus, exception); - } - - @Override - public boolean terminate(Map jobs) { - boolean ok = true; - for (Job job : jobs.values()) { - if (job.getJobData() == null || job.getJobData().getJobStatus() == null) { - continue; - } - String jobId = job.getJobData().getJobID() == null ? "NULL" : job.getJobData().getJobID(); - if (job instanceof TerminatableJob) { - try { - ((TerminatableJob) job).terminate(); - } catch (Exception e) { - System.out.println( - "Failed to kill job: jobId=" + jobId + ". " + ExceptionUtils.getStackTrace(e)); - } - if (!job.getJobData().getJobStatus().isJobCancelled() - || !job.getJobData().getJobStatus().isJobFailure()) { - ok = false; - System.out.println( - "Failed to kill job: jobId=" - + jobId - + ", current status: " - + job.getJobData().getJobStatus().toString()); - } else { - System.out.println( - "Successfully killed job: jobId=" - + jobId - + ", current status: " - + job.getJobData().getJobStatus().toString()); - } - } else { - System.out.println("Job \"" + jobId + "\"" + "is not terminatable"); - } - } - return ok; - } - - private void ExecWithAsyncBackend(Job job) { - - if (!(job instanceof AsyncBackendJob)) { - throw new LinkisClientExecutionException( - "EXE0002", - ErrorLevel.ERROR, - CommonErrMsg.ExecutionErr, - "job is not instance of AsyncBackendJob"); - } - AsyncBackendJob submitJob = (AsyncBackendJob) job; - - submitJob.submit(); - CommonUtils.doSleepQuietly(CommonConstants.JOB_QUERY_SLEEP_MILLS); - - if (!submitJob.getJobData().getJobStatus().isJobSubmitted()) { - throw new LinkisClientExecutionException( - "EXE0005", - ErrorLevel.ERROR, - CommonErrMsg.ExecutionErr, - "Retry exhausted checking job submission. Job is probably not submitted"); - } else { - // Output that job is submitted - StringBuilder infoBuilder = new StringBuilder(); - infoBuilder.append("Job is successfully submitted!").append(System.lineSeparator()); - LogUtils.getInformationLogger().info(infoBuilder.toString()); - } - - if (job instanceof LogAccessibleJob) { - /* - Non-blocking, call if back-end supports it - */ - ((LogAccessibleJob) job).startRetrieveLog(); - } - - submitJob.waitJobComplete(); - - if (submitJob.getJobData().getJobStatus().isJobFinishedState()) { - if (job instanceof ResultAccessibleJob) { - /* - Non-blocking, call if back-end supports it - */ - ((ResultAccessibleJob) job).startRetrieveResult(); - } - } - } -} diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/job/AbstractJob.java b/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/job/AbstractJob.java deleted file mode 100644 index 0de5e27ce6..0000000000 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/job/AbstractJob.java +++ /dev/null @@ -1,77 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.cli.core.interactor.job; - -import org.apache.linkis.cli.common.entity.command.CmdType; -import org.apache.linkis.cli.common.entity.job.Job; -import org.apache.linkis.cli.common.entity.job.JobSubType; -import org.apache.linkis.cli.common.entity.operator.JobOperator; -import org.apache.linkis.cli.common.entity.present.PresentWay; - -public abstract class AbstractJob implements Job { - protected String cid; - protected CmdType cmdType; - protected JobSubType subType; - protected JobOperator operator; - private PresentWay presentWay; - - @Override - public String getCid() { - return cid; - } - - public void setCid(String cid) { - this.cid = cid; - } - - @Override - public CmdType getCmdType() { - return cmdType; - } - - public void setCmdType(CmdType cmdType) { - this.cmdType = cmdType; - } - - @Override - public JobSubType getSubType() { - return subType; - } - - public void setSubType(JobSubType subType) { - this.subType = subType; - } - - @Override - public JobOperator getJobOperator() { - return operator; - } - - public void setOperator(JobOperator operator) { - this.operator = operator; - } - - @Override - public PresentWay getPresentWay() { - return presentWay; - } - - public void setPresentWay(PresentWay presentWay) { - this.presentWay = presentWay; - } -} diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/job/AsyncBackendJob.java b/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/job/AsyncBackendJob.java deleted file mode 100644 index 636a361cad..0000000000 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/job/AsyncBackendJob.java +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.cli.core.interactor.job; - -import org.apache.linkis.cli.common.entity.job.Job; -import org.apache.linkis.cli.common.entity.job.JobData; -import org.apache.linkis.cli.common.exception.LinkisClientRuntimeException; - -/** - * If backend supports async-submission, i.e. submit job and then return while job is running. Then - * implement this. Note that all results return by server should be returned but stored in {@link - * JobData} - */ -public interface AsyncBackendJob extends Job { - - void submit() throws LinkisClientRuntimeException; - - void updateJobStatus() throws LinkisClientRuntimeException; - - void waitJobComplete() throws LinkisClientRuntimeException; - - void terminate() throws LinkisClientRuntimeException; -} diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/job/JobBuilder.java b/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/job/JobBuilder.java deleted file mode 100644 index 566fe3b279..0000000000 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/job/JobBuilder.java +++ /dev/null @@ -1,52 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.cli.core.interactor.job; - -import org.apache.linkis.cli.common.entity.command.CmdType; -import org.apache.linkis.cli.common.entity.job.JobData; -import org.apache.linkis.cli.common.entity.job.JobDescription; -import org.apache.linkis.cli.common.entity.job.JobSubType; -import org.apache.linkis.cli.common.entity.operator.JobOperator; -import org.apache.linkis.cli.common.entity.present.PresentWay; -import org.apache.linkis.cli.core.builder.BuildableByVarAccess; - -public abstract class JobBuilder extends BuildableByVarAccess { - - public JobBuilder setCid(String cid) { - targetObj.setCid(cid); - return this; - } - - public JobBuilder setCmdType(CmdType cmdType) { - targetObj.setCmdType(cmdType); - return this; - } - - public JobBuilder setJobSubType(JobSubType subType) { - targetObj.setSubType(subType); - return this; - } - - protected abstract JobDescription buildJobDesc(); - - protected abstract JobData buildJobData(); - - protected abstract JobOperator buildJobOperator(); - - protected abstract PresentWay buildPresentWay(); -} diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/job/LogAccessibleJob.java b/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/job/LogAccessibleJob.java deleted file mode 100644 index d372c444a3..0000000000 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/job/LogAccessibleJob.java +++ /dev/null @@ -1,30 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.cli.core.interactor.job; - -import org.apache.linkis.cli.common.entity.job.Job; -import org.apache.linkis.cli.common.entity.job.JobData; - -/** If backend supports extracting log while job is running, then implement this interface */ -public interface LogAccessibleJob extends Job { - /** - * Retrieve log. This methods should be non-blocking and log should be appended into {@link - * JobData} - */ - void startRetrieveLog(); -} diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/job/ManagableBackendJob.java b/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/job/ManagableBackendJob.java deleted file mode 100644 index 204134110b..0000000000 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/job/ManagableBackendJob.java +++ /dev/null @@ -1,42 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.cli.core.interactor.job; - -import org.apache.linkis.cli.common.entity.job.Job; -import org.apache.linkis.cli.common.entity.job.JobStatus; -import org.apache.linkis.cli.common.entity.job.JobSubType; -import org.apache.linkis.cli.common.exception.LinkisClientRuntimeException; - -/** - * Backend support job-management, e.g. get status, get log, kill job etc. Then implement this - * interface. - */ -public interface ManagableBackendJob extends Job { - /** - * since job management can diverge, we decide to assign it to lower-level implementation. - * implementation should use {@link JobSubType} to decide which action to take, hence {@link - * JobSubType} should not be null - */ - void doManage() throws LinkisClientRuntimeException; - - /** - * if execution is success. This can be different from {@link JobStatus} e.g. query job status, - * job may be FAIL but execution is a asuccess - */ - boolean isSuccess(); -} diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/job/ResultAccessibleJob.java b/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/job/ResultAccessibleJob.java deleted file mode 100644 index 564ed03e10..0000000000 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/job/ResultAccessibleJob.java +++ /dev/null @@ -1,29 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.cli.core.interactor.job; - -import org.apache.linkis.cli.common.entity.job.Job; -import org.apache.linkis.cli.common.entity.job.JobData; - -public interface ResultAccessibleJob extends Job { - /** - * Retrieve reault-set. This methods should be non-blocking and result should be appended into - * {@link JobData} - */ - void startRetrieveResult(); -} diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/job/SyncBackendJob.java b/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/job/SyncBackendJob.java deleted file mode 100644 index d18b81b91d..0000000000 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/job/SyncBackendJob.java +++ /dev/null @@ -1,29 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.cli.core.interactor.job; - -import org.apache.linkis.cli.common.entity.job.Job; -import org.apache.linkis.cli.common.exception.LinkisClientRuntimeException; - -/** - * Backend only supports sync-submission, i.e. submit and wait till job finish and get result in one - * call, then implement this interface. - */ -public interface SyncBackendJob extends Job { - void submitAndGetResult() throws LinkisClientRuntimeException; -} diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/job/TerminatableJob.java b/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/job/TerminatableJob.java deleted file mode 100644 index ff7e832724..0000000000 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/job/TerminatableJob.java +++ /dev/null @@ -1,29 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.cli.core.interactor.job; - -import org.apache.linkis.cli.common.entity.job.Job; -import org.apache.linkis.cli.common.exception.LinkisClientRuntimeException; - -/** Backend supports killing job */ -public interface TerminatableJob extends Job { - /** for jobs that starts a new thread */ - TerminateToken getTerminateToken(); - - void terminate() throws LinkisClientRuntimeException; -} diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/job/TerminateToken.java b/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/job/TerminateToken.java deleted file mode 100644 index e8d81a430d..0000000000 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/job/TerminateToken.java +++ /dev/null @@ -1,32 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.cli.core.interactor.job; - -import java.util.concurrent.atomic.AtomicBoolean; - -public class TerminateToken { - private AtomicBoolean shouldTerminate = new AtomicBoolean(false); - - public boolean shouldTerminate() { - return shouldTerminate.get(); - } - - public void setTerminate() { - shouldTerminate.set(true); - } -} diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/properties/PropertiesLoader.java b/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/properties/PropertiesLoader.java deleted file mode 100644 index df79c8262e..0000000000 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/properties/PropertiesLoader.java +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.cli.core.interactor.properties; - -import org.apache.linkis.cli.common.entity.properties.ClientProperties; -import org.apache.linkis.cli.core.interactor.properties.reader.PropertiesReader; - -public interface PropertiesLoader { - PropertiesLoader setPropertiesReaders(PropertiesReader[] readers); - - PropertiesLoader addPropertiesReader(PropertiesReader reader); - - PropertiesLoader addPropertiesReaders(PropertiesReader[] readers); - - void removePropertiesReader(String identifier); - - ClientProperties[] loadProperties(); - - void checkInit(); -} diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/properties/reader/SysEnvReader.java b/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/properties/reader/SysEnvReader.java deleted file mode 100644 index b89ebe9755..0000000000 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/properties/reader/SysEnvReader.java +++ /dev/null @@ -1,64 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.cli.core.interactor.properties.reader; - -import org.apache.linkis.cli.core.constants.CommonConstants; - -import java.util.Properties; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public class SysEnvReader implements PropertiesReader { - private static final Logger logger = LoggerFactory.getLogger(SysEnvReader.class); - private String propsId = CommonConstants.SYSTEM_ENV_IDENTIFIER; - private String propsPath = "SYSTEM"; - - @Override - public String getPropsId() { - return propsId; - } - - @Override - public PropertiesReader setPropsId(String identifier) { - this.propsId = identifier; - return this; - } - - @Override - public String getPropsPath() { - return propsPath; - } - - @Override - public PropertiesReader setPropsPath(String propertiesPath) { - this.propsPath = propertiesPath; - return this; - } - - @Override - public Properties getProperties() { - checkInit(); - Properties props = new Properties(); - props.putAll(System.getenv()); - return props; - } - - @Override - public void checkInit() {} -} diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/properties/reader/SysPropsReader.java b/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/properties/reader/SysPropsReader.java deleted file mode 100644 index bf4bbc0270..0000000000 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/properties/reader/SysPropsReader.java +++ /dev/null @@ -1,62 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.cli.core.interactor.properties.reader; - -import org.apache.linkis.cli.core.constants.CommonConstants; - -import java.util.Properties; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public class SysPropsReader implements PropertiesReader { - private static final Logger logger = LoggerFactory.getLogger(SysPropsReader.class); - private String propsId = CommonConstants.SYSTEM_PROPERTIES_IDENTIFIER; - private String propsPath = "SYSTEM"; - - @Override - public String getPropsId() { - return propsId; - } - - @Override - public PropertiesReader setPropsId(String identifier) { - this.propsId = identifier; - return this; - } - - @Override - public String getPropsPath() { - return propsPath; - } - - @Override - public PropertiesReader setPropsPath(String propertiesPath) { - this.propsPath = propertiesPath; - return this; - } - - @Override - public Properties getProperties() { - checkInit(); - return System.getProperties(); - } - - @Override - public void checkInit() {} -} diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/result/DefaultResultHandler.java b/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/result/DefaultResultHandler.java deleted file mode 100644 index 836aeb7ad8..0000000000 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/result/DefaultResultHandler.java +++ /dev/null @@ -1,39 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.cli.core.interactor.result; - -import org.apache.linkis.cli.common.entity.result.ExecutionResult; -import org.apache.linkis.cli.common.entity.result.ResultHandler; -import org.apache.linkis.cli.core.constants.CommonConstants; -import org.apache.linkis.cli.core.utils.LogUtils; - -import static java.lang.System.exit; - -/** exit -1 when failure and exit 0 when success */ -public class DefaultResultHandler implements ResultHandler { - @Override - public void process(ExecutionResult executionResult) { - if (executionResult.getExecutionStatus() == ExecutionStatusEnum.SUCCEED) { - LogUtils.getPlaintTextLogger().info(CommonConstants.SUCCESS_INDICATOR); - exit(0); - } else { - LogUtils.getPlaintTextLogger().info(CommonConstants.FAILURE_INDICATOR); - exit(-1); - } - } -} diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/result/ExecutionResultImpl.java b/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/result/ExecutionResultImpl.java deleted file mode 100644 index b145ba9c8b..0000000000 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/result/ExecutionResultImpl.java +++ /dev/null @@ -1,68 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.cli.core.interactor.result; - -import org.apache.linkis.cli.common.entity.job.Job; -import org.apache.linkis.cli.common.entity.result.ExecutionResult; -import org.apache.linkis.cli.common.entity.result.ExecutionStatus; - -import java.util.Map; - -public class ExecutionResultImpl implements ExecutionResult { - - Map jobsMap; - ExecutionStatus executionStatus; - Exception exception; - - public ExecutionResultImpl(Map jobsMap, ExecutionStatus executionStatus) { - this.jobsMap = jobsMap; - this.executionStatus = executionStatus; - } - - public ExecutionResultImpl( - Map jobsMap, ExecutionStatus executionStatus, Exception exception) { - this.jobsMap = jobsMap; - this.executionStatus = executionStatus; - this.exception = exception; - } - - @Override - public Map getJobs() { - return jobsMap; - } - - @Override - public ExecutionStatus getExecutionStatus() { - return this.executionStatus; - } - - @Override - public void setExecutionStatus(ExecutionStatus executionStatus) { - this.executionStatus = executionStatus; - } - - @Override - public Exception getException() { - return this.exception; - } - - @Override - public void setException(Exception exception) { - this.exception = exception; - } -} diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/result/ExecutionStatusEnum.java b/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/result/ExecutionStatusEnum.java deleted file mode 100644 index 0c4375b3d1..0000000000 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/result/ExecutionStatusEnum.java +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.cli.core.interactor.result; - -import org.apache.linkis.cli.common.entity.result.ExecutionStatus; - -public enum ExecutionStatusEnum implements ExecutionStatus { - UNDEFINED("Inited", 1), - SUCCEED("Succeed", 2), - FAILED("Failed", 3); - - private String name; - private int id; - - ExecutionStatusEnum(String name, int id) { - this.name = name; - this.id = id; - } -} diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/result/PresentResultHandler.java b/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/result/PresentResultHandler.java deleted file mode 100644 index 5a3fbfa4a2..0000000000 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/result/PresentResultHandler.java +++ /dev/null @@ -1,78 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.cli.core.interactor.result; - -import org.apache.linkis.cli.common.entity.job.Job; -import org.apache.linkis.cli.common.entity.present.Model; -import org.apache.linkis.cli.common.entity.present.Presenter; -import org.apache.linkis.cli.common.entity.result.ExecutionResult; -import org.apache.linkis.cli.common.entity.result.ResultHandler; -import org.apache.linkis.cli.common.exception.error.ErrorLevel; -import org.apache.linkis.cli.core.exception.LinkisClientExecutionException; -import org.apache.linkis.cli.core.exception.error.CommonErrMsg; - -import java.util.Map; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public class PresentResultHandler implements ResultHandler { - private static Logger logger = LoggerFactory.getLogger(PresentResultHandler.class); - Presenter presenter; - Model model; - - public void checkInit() { - if (presenter == null || model == null) { - throw new LinkisClientExecutionException( - "EXE0031", - ErrorLevel.ERROR, - CommonErrMsg.ExecutionResultErr, - "Presenter or model is null"); - } - } - - public void setPresenter(Presenter presenter) { - this.presenter = presenter; - } - - public void setModel(Model model) { - this.model = model; - } - - @Override - public void process(ExecutionResult executionResult) { - checkInit(); - Map jobs = executionResult.getJobs(); - // Probably need modification if we further want multiple-jobs support - // but we probably don't want to support that - if (jobs != null) { - for (Job job : jobs.values()) { - if (job != null) { - model.buildModel(job.getJobData()); - } - try { - presenter.present(model, job.getPresentWay()); - } catch (Exception e) { - logger.error("Execution failed because exception thrown when presenting data.", e); - executionResult.setExecutionStatus(ExecutionStatusEnum.FAILED); - executionResult.setException(e); - } - } - } - } -} diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/validate/PropsValidator.java b/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/validate/PropsValidator.java deleted file mode 100644 index 6231f4eb88..0000000000 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/validate/PropsValidator.java +++ /dev/null @@ -1,31 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.cli.core.interactor.validate; - -import org.apache.linkis.cli.common.entity.properties.ClientProperties; -import org.apache.linkis.cli.common.entity.validate.Validator; -import org.apache.linkis.cli.common.exception.LinkisClientRuntimeException; - -public class PropsValidator implements Validator { - @Override - public void doValidation(Object input) throws LinkisClientRuntimeException { - if (!(input instanceof ClientProperties)) { - // TODO:throw - } - } -} diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/var/SysVarAccess.java b/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/var/SysVarAccess.java deleted file mode 100644 index d291e1bda5..0000000000 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/var/SysVarAccess.java +++ /dev/null @@ -1,114 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.cli.core.interactor.var; - -import org.apache.linkis.cli.common.entity.properties.ClientProperties; -import org.apache.linkis.cli.common.entity.var.VarAccess; -import org.apache.linkis.cli.common.exception.error.ErrorLevel; -import org.apache.linkis.cli.core.exception.VarAccessException; -import org.apache.linkis.cli.core.exception.error.CommonErrMsg; - -import java.util.ArrayList; -import java.util.List; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public class SysVarAccess implements VarAccess { - private static Logger logger = LoggerFactory.getLogger(SysVarAccess.class); - private ClientProperties sysProp; - private ClientProperties sysEnv; - - public SysVarAccess setSysProp(ClientProperties sysProp) { - this.sysProp = sysProp; - return this; - } - - public ClientProperties getSysProp(String identifier) { - return this.sysProp; - } - - public SysVarAccess setSysEnv(ClientProperties sysEnv) { - this.sysEnv = sysEnv; - return this; - } - - public ClientProperties getSysEnv(String identifier) { - return this.sysEnv; - } - - @Override - public void checkInit() { - if (this.sysProp == null && this.sysEnv == null) { - throw new VarAccessException( - "VA0001", - ErrorLevel.ERROR, - CommonErrMsg.VarAccessInitErr, - "sys_prop and sys_env are both null"); - } - } - - @Override - public T getVar(Class clazz, String key) { - checkInit(); - if (clazz != String.class) { - // throw exception - } - Object o1 = sysProp.get(key); - Object o2 = sysEnv.get(key); - if (o1 != null && o2 != null) { - throw new VarAccessException( - "VA0002", - ErrorLevel.WARN, - CommonErrMsg.VarAccessErr, - "same key occurred in sys_prop and sys_env. will use sys_prop"); - } - Object ret = o1 != null ? o1 : o2; - return clazz.cast(ret); - } - - @Override - public T getVarOrDefault(Class clazz, String key, T defaultValue) { - T ret = getVar(clazz, key); - if (ret == null) { - ret = defaultValue; - } - return ret; - } - - @Override - public String[] getAllVarKeys() { - List varKeys = new ArrayList<>(); - if (sysProp != null) { - for (Object key : sysProp.keySet()) { - varKeys.add((String) key); - } - } - if (sysEnv != null) { - for (Object key : sysEnv.keySet()) { - varKeys.add((String) key); - } - } - return varKeys.toArray(new String[varKeys.size()]); - } - - @Override - public boolean hasVar(String key) { - return sysEnv.containsKey(key) || sysProp.containsKey(key); - } -} diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/operator/JobOperatorFactory.java b/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/operator/JobOperatorFactory.java deleted file mode 100644 index dd6031d66a..0000000000 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/operator/JobOperatorFactory.java +++ /dev/null @@ -1,93 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.cli.core.operator; - -import org.apache.linkis.cli.common.entity.operator.JobOperator; -import org.apache.linkis.cli.common.exception.error.ErrorLevel; -import org.apache.linkis.cli.core.exception.LinkisClientExecutionException; -import org.apache.linkis.cli.core.exception.error.CommonErrMsg; - -import java.util.Map; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.locks.ReentrantLock; - -public class JobOperatorFactory { - private static Map builderMap = new ConcurrentHashMap<>(); - - private static Map instanceMap = new ConcurrentHashMap<>(); // for singleton - private static Map lockMap = new ConcurrentHashMap<>(); // for singleton - - public static synchronized void register(String name, JobOperatorBuilder builder) - throws Exception { - if (builderMap.containsKey(name) - || lockMap.containsKey(name) - || instanceMap.containsKey(name)) { - throw new LinkisClientExecutionException( - "EXE0027", - ErrorLevel.ERROR, - CommonErrMsg.ExecutionInitErr, - "Attempting to register a duplicate jobOperator, name: " + name); - } - builderMap.put(name, builder); - lockMap.put(name, new ReentrantLock()); - } - - public static synchronized void remove(String name) { - builderMap.remove(name); - instanceMap.remove(name); - lockMap.remove(name); - } - - public static JobOperator getReusable(String name) throws Exception { - JobOperatorBuilder builder = builderMap.get(name); - ReentrantLock lock = lockMap.get(name); - JobOperator instance = instanceMap.get(name); - if (lock == null || builder == null) { - throw new LinkisClientExecutionException( - "EXE0028", - ErrorLevel.ERROR, - CommonErrMsg.ExecutionInitErr, - "Failed to get a reusable joboperator, name: " + name); - } - if (instance == null) { - boolean ok = lock.tryLock(500, TimeUnit.MILLISECONDS); - if (!ok) { - throw new LinkisClientExecutionException( - "EXE0028", - ErrorLevel.ERROR, - CommonErrMsg.ExecutionInitErr, - "Failed to get a reusable joboperator, name: " + name); - } - if (instance == null) { - instance = builder.build(); - instanceMap.put(name, instance); - } - lock.unlock(); - } - return instance; - } - - public static JobOperator getNew(String name) throws Exception { - JobOperatorBuilder builder = builderMap.get(name); - if (builder == null) { - throw new Exception("TODO"); // TODO - } - return builder.build(); - } -} diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/present/DefaultStdOutPresenter.java b/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/present/DefaultStdOutPresenter.java deleted file mode 100644 index 7ed2307520..0000000000 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/present/DefaultStdOutPresenter.java +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.cli.core.present; - -import org.apache.linkis.cli.common.entity.present.Model; -import org.apache.linkis.cli.common.entity.present.PresentWay; -import org.apache.linkis.cli.common.entity.present.Presenter; -import org.apache.linkis.cli.core.present.display.DisplayOperator; -import org.apache.linkis.cli.core.present.display.StdOutWriter; -import org.apache.linkis.cli.core.present.display.data.StdoutDisplayData; -import org.apache.linkis.cli.core.utils.CommonUtils; - -public class DefaultStdOutPresenter implements Presenter { - DisplayOperator driver = new StdOutWriter(); - - @Override - public void present(Model model, PresentWay presentWay) { - driver.doOutput(new StdoutDisplayData(CommonUtils.GSON.toJson(model))); - } -} diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/present/PresentMode.java b/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/present/PresentMode.java deleted file mode 100644 index 6d26a2eb49..0000000000 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/present/PresentMode.java +++ /dev/null @@ -1,22 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.cli.core.present; - -public interface PresentMode { - String getName(); -} diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/present/PresentWayImpl.java b/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/present/PresentWayImpl.java deleted file mode 100644 index 6fbe4e1ec0..0000000000 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/present/PresentWayImpl.java +++ /dev/null @@ -1,51 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.cli.core.present; - -import org.apache.linkis.cli.common.entity.present.PresentWay; - -public class PresentWayImpl implements PresentWay { - - private PresentMode mode; - private String path; - private boolean displayMetaAndLogo = true; - - public String getPath() { - return path; - } - - public void setPath(String path) { - this.path = path; - } - - public boolean isDisplayMetaAndLogo() { - return displayMetaAndLogo; - } - - public void setDisplayMetaAndLogo(boolean displayMetaAndLogo) { - this.displayMetaAndLogo = displayMetaAndLogo; - } - - public PresentMode getMode() { - return mode; - } - - public void setMode(PresentMode mode) { - this.mode = mode; - } -} diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/present/display/DisplayOperFactory.java b/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/present/display/DisplayOperFactory.java deleted file mode 100644 index 9f584f4505..0000000000 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/present/display/DisplayOperFactory.java +++ /dev/null @@ -1,50 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.cli.core.present.display; - -import org.apache.linkis.cli.common.exception.error.ErrorLevel; -import org.apache.linkis.cli.core.exception.PresenterException; -import org.apache.linkis.cli.core.exception.error.CommonErrMsg; -import org.apache.linkis.cli.core.present.PresentMode; - -import java.util.Map; -import java.util.concurrent.ConcurrentHashMap; - -public class DisplayOperFactory { - private static final Map operatorMap = new ConcurrentHashMap<>(); - - public static synchronized void register(PresentMode mode, DisplayOperator operator) - throws Exception { - if (operatorMap.containsKey(mode.getName())) { - throw new PresenterException( - "PST0012", - ErrorLevel.ERROR, - CommonErrMsg.PresenterInitErr, - "Attempting to register a duplicate DisplayOperator, name: " + mode.getName()); - } - operatorMap.put(mode.getName(), operator); - } - - public static synchronized void remove(PresentMode mode) { - operatorMap.remove(mode.getName()); - } - - public static DisplayOperator getDisplayOper(PresentMode mode) { - return operatorMap.get(mode.getName()); - } -} diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/present/display/StdOutWriter.java b/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/present/display/StdOutWriter.java deleted file mode 100644 index bc91340dbf..0000000000 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/present/display/StdOutWriter.java +++ /dev/null @@ -1,43 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.cli.core.present.display; - -import org.apache.linkis.cli.common.exception.error.ErrorLevel; -import org.apache.linkis.cli.core.exception.PresenterException; -import org.apache.linkis.cli.core.exception.error.CommonErrMsg; -import org.apache.linkis.cli.core.present.display.data.DisplayData; -import org.apache.linkis.cli.core.present.display.data.StdoutDisplayData; -import org.apache.linkis.cli.core.utils.LogUtils; - -import org.slf4j.Logger; - -public class StdOutWriter implements DisplayOperator { - @Override - public void doOutput(DisplayData data) { - if (!(data instanceof StdoutDisplayData)) { - throw new PresenterException( - "PST0008", - ErrorLevel.ERROR, - CommonErrMsg.PresentDriverErr, - "input data is not instance of StdoutDisplayData"); - } - String content = ((StdoutDisplayData) data).getContent(); - Logger logger = LogUtils.getPlaintTextLogger(); - logger.info(content); - } -} diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/present/display/data/DisplayData.java b/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/present/display/data/DisplayData.java deleted file mode 100644 index 38463b7269..0000000000 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/present/display/data/DisplayData.java +++ /dev/null @@ -1,20 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.cli.core.present.display.data; - -public interface DisplayData {} diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/present/display/data/FileDisplayData.java b/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/present/display/data/FileDisplayData.java deleted file mode 100644 index 607e05fd86..0000000000 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/present/display/data/FileDisplayData.java +++ /dev/null @@ -1,47 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.cli.core.present.display.data; - -public class FileDisplayData extends StdoutDisplayData implements DisplayData { - private String pathName; - private String fileName; - private Boolean createNewFile; - - public FileDisplayData(String pathName, String fileName, String content, Boolean createNewFile) { - super(content); - this.pathName = pathName; - this.fileName = fileName; - this.createNewFile = createNewFile; - } - - public String getPathName() { - return pathName; - } - - public String getContent() { - return super.getContent(); - } - - public boolean getCreateNewFile() { - return createNewFile; - } - - public String getFileName() { - return fileName; - } -} diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/present/display/data/StdoutDisplayData.java b/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/present/display/data/StdoutDisplayData.java deleted file mode 100644 index e332256417..0000000000 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/present/display/data/StdoutDisplayData.java +++ /dev/null @@ -1,30 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.cli.core.present.display.data; - -public class StdoutDisplayData implements DisplayData { - private String content; - - public StdoutDisplayData(String content) { - this.content = content; - } - - public String getContent() { - return content; - } -} diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/utils/CommonUtils.java b/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/utils/CommonUtils.java deleted file mode 100644 index 8d1a3b8ac9..0000000000 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/utils/CommonUtils.java +++ /dev/null @@ -1,82 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.cli.core.utils; - -import org.apache.commons.lang3.StringUtils; - -import java.util.HashMap; -import java.util.Map; - -import com.google.gson.Gson; -import com.google.gson.GsonBuilder; - -public class CommonUtils { - - public static final Gson GSON = - new GsonBuilder().setPrettyPrinting().disableHtmlEscaping().create(); - - public static T castStringToAny(Class clazz, String val) { - if (StringUtils.isBlank(val)) { - return null; - } - T ret = null; - if (clazz == Object.class) { - ret = clazz.cast(val); - } else if (clazz == String.class) { - ret = clazz.cast(val); - } else if (clazz == Integer.class) { - ret = clazz.cast(Integer.parseInt(val)); - } else if (clazz == Double.class) { - ret = clazz.cast(Double.parseDouble(val)); - } else if (clazz == Float.class) { - ret = clazz.cast(Float.parseFloat(val)); - } else if (clazz == Long.class) { - ret = clazz.cast(Long.parseLong(val)); - } else if (clazz == Boolean.class) { - ret = clazz.cast(Boolean.parseBoolean(val)); - } - return ret; - } - - public static void doSleepQuietly(Long sleepMills) { - try { - Thread.sleep(sleepMills); - } catch (Exception ignore) { - // ignored - } - } - - public static Map parseKVStringToMap(String kvStr, String separator) { - if (StringUtils.isBlank(separator)) { - separator = ","; - } - if (StringUtils.isBlank(kvStr)) { - return null; - } - Map argsProps = new HashMap<>(); - String[] args = StringUtils.splitByWholeSeparator(kvStr, separator); - for (String arg : args) { - int index = arg.indexOf("="); - if (index != -1) { - argsProps.put(arg.substring(0, index).trim(), arg.substring(index + 1).trim()); - } - } - - return argsProps; - } -} diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/test/java/org/apache/linkis/cli/core/constants/CommonConstantsTest.java b/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/test/java/org/apache/linkis/cli/core/constants/CommonConstantsTest.java deleted file mode 100644 index a1fb04f01d..0000000000 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/test/java/org/apache/linkis/cli/core/constants/CommonConstantsTest.java +++ /dev/null @@ -1,60 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.cli.core.constants; - -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.DisplayName; -import org.junit.jupiter.api.Test; - -public class CommonConstantsTest { - - @Test - @DisplayName("constTest") - public void constTest() { - - Long jobQuerySleepMills = CommonConstants.JOB_QUERY_SLEEP_MILLS; - Integer requestMaxRetryTime = CommonConstants.REQUEST_MAX_RETRY_TIME; - String universalSubcmd = CommonConstants.UNIVERSAL_SUBCMD; - String universalSubcmdDesc = CommonConstants.UNIVERSAL_SUBCMD_DESC; - String successIndicator = CommonConstants.SUCCESS_INDICATOR; - String failureIndicator = CommonConstants.FAILURE_INDICATOR; - String arraySeq = CommonConstants.ARRAY_SEQ; - String arraySeqRegex = CommonConstants.ARRAY_SEQ_REGEX; - int maxNumOfCommandArguements = CommonConstants.MAX_NUM_OF_COMMAND_ARGUEMENTS; - String configDir = CommonConstants.CONFIG_DIR; - String[] configExtension = CommonConstants.CONFIG_EXTENSION; - String systemPropertiesIdentifier = CommonConstants.SYSTEM_PROPERTIES_IDENTIFIER; - String systemEnvIdentifier = CommonConstants.SYSTEM_ENV_IDENTIFIER; - - Assertions.assertTrue(2000l == jobQuerySleepMills); - Assertions.assertTrue(3 == requestMaxRetryTime); - Assertions.assertEquals("linkis-cli", universalSubcmd); - Assertions.assertEquals( - "command for all types of jobs supported by Linkis", universalSubcmdDesc); - - Assertions.assertEquals("############Execute Success!!!########", successIndicator); - Assertions.assertEquals("############Execute Error!!!########", failureIndicator); - Assertions.assertEquals("@#@", arraySeq); - Assertions.assertEquals("(?=([^\"]*\"[^\"]*\")*[^\"]*$)", arraySeqRegex); - Assertions.assertTrue(10 == maxNumOfCommandArguements); - Assertions.assertEquals("config.path", configDir); - Assertions.assertTrue(configExtension.length == 1); - Assertions.assertEquals("SYS_PROP", systemPropertiesIdentifier); - Assertions.assertEquals("SYS_ENV", systemEnvIdentifier); - } -} diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/test/java/org/apache/linkis/cli/core/constants/TestConstants.java b/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/test/java/org/apache/linkis/cli/core/constants/TestConstants.java deleted file mode 100644 index 307369054b..0000000000 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/test/java/org/apache/linkis/cli/core/constants/TestConstants.java +++ /dev/null @@ -1,100 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.cli.core.constants; - -public class TestConstants { - - public static final String PRIMARY = "test.primary"; - public static final String PRIMARY_DESC = " command"; - - // for command name - public static final String SPARK = "spark"; - public static final String HIVE = "hive"; - - // for command description - public static final String SPARK_DESC = "Execute sql with spark"; - public static final String HIVE_DESC = "Execute hive sql"; - public static final String JOB_DESC = ""; - - // Common command params - public static final String PARAM_COMMON_FILE = "param.common.file"; - public static final String PARAM_COMMON_CMD = "param.common.command"; - public static final String PARAM_COMMON_OUTPUT = "param.common.output"; - public static final String PARAM_COMMON_ARGS = "param.common.args"; - public static final String PARAM_COMMON_SPLIT = "param.common.split"; - public static final String PARAM_COMMON_OTHER_KV = - "param.common.other.kv"; // for customize some parameters for some commands. Has to be - // in the for of kv-pairs separated by ',' - public static final String PARAM_YARN_QUEUE = "param.yarn.queue"; - - // for job command - public static final String PARAM_JOB_TYPE = "param.job.type"; - public static final String PARAM_JOB_ID = "param.job.id"; - public static final String PARAM_FORCE_KILL = "param.force.kill"; - public static final String LINKIS_JOBID_PREFIX = "lks_"; - - public static final String PARAM_MAPPER_KV_STRING = - "param.mapper.kv.string"; // Rules for ParamKeyMapper. format: - // key1=value1,key2=value2... - public static final String YARN_QUEUE = "wds.linkis.yarnqueue"; - public static final String YARN_QUEUE_DEFAULT = "default"; - public static final String YARN_QUEUE_CORE_MAX = "wds.linkis.yarnqueue.cores.max"; - public static final int YARN_QUEUE_CORE_MAX_DEFAULT = 150; - public static final String YARN_QUEUE_MEM_MAX = "wds.linkis.yarnqueue.memory.max"; - public static final String YARN_QUEUE_MEM_MAX_DEFAULT = "300G"; - public static final String PREHEATING_TIME = "wds.linkis.preheating.time"; - public static final String PREHEATING_TIME_DEFAULT = "9:00"; - public static final String TMPFILE_CLEAN_TIME = "wds.linkis.tmpfile.clean.time"; - public static final String TMPFILE_CLEAN_TIME_DEFAULT = "10:00"; - public static final String LINKIS_INSTANCE = "wds.linkis.instance"; - public static final int LINKIS_INSTANCE_DEFAULT = 10; - public static final String LINKIS_CLIENT_MEMORY_MAX = "wds.linkis.client.memory.max"; - public static final String LINKIS_CLIENT_MEMORY_MAX_DEFAULT = "20G"; - - // Common - public static final String LINKIS_NULL_VALUE = ""; - - public static final String SPARK_CMD = "spark"; - - public static final String PARAM_SPARK_NAME = "param.spark.name"; - public static final String PARAM_SPARK_HIVECONF = "param.spark.hiveconf"; - public static final String PARAM_SPARK_NUM_EXECUTORS = "param.spark.num.executors"; - public static final String PARAM_SPARK_EXECUTOR_CORES = "param.spark.executor.cores"; - public static final String PARAM_SPARK_EXECUTOR_MEMORY = "param.spark.executor.memory"; - public static final String PARAM_SPARK_SHUFFLE_PARTITIONS = "param.spark.shuffle.partitions"; - public static final String PARAM_SPARK_RUNTYPE = "param.spark.runtype"; - - public static final String LINKIS_SPARK_NAME = "appName"; - public static final String LINKIS_SPARK_NUM_EXECUTORS = "spark.executor.instances"; - public static final String LINKIS_SPARK_EXECUTOR_CORES = "spark.executor.cores"; - public static final String LINKIS_SPARK_EXECUTOR_MEMORY = "spark.executor.memory"; - public static final String LINKIS_SPARK_SHUFFLE_PARTITIONS = "spark.sql.shuffle.partitions"; - - public static final String PARAM_DB = "param.primary.database"; - public static final String PARAM_PROXY = "param.primary.proxy"; - public static final String PARAM_USER = "param.primary.user"; - public static final String PARAM_USR_CONF = "param.primary.user.conf"; - public static final String PARAM_PASSWORD = "param.primary.password"; - public static final String PARAM_SYNC_KEY = "param.primary.synckey"; - public static final String PARAM_PROXY_USER = "param.primary.proxyUser"; - public static final String PARAM_HELP = "param.help"; - public static final String PARAM_REAL_NAME = "param.primary.realName"; - public static final String PARAM_PIN_TOKEN = "param.primary.pinToken"; - - public static final String PARAM_PROPERTIES = "params.properties"; -} diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/test/java/org/apache/linkis/cli/core/exception/error/CommonErrMsgTest.java b/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/test/java/org/apache/linkis/cli/core/exception/error/CommonErrMsgTest.java deleted file mode 100644 index cdd51f4aea..0000000000 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/test/java/org/apache/linkis/cli/core/exception/error/CommonErrMsgTest.java +++ /dev/null @@ -1,41 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.cli.core.exception.error; - -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.DisplayName; -import org.junit.jupiter.api.Test; - -public class CommonErrMsgTest { - - @Test - @DisplayName("enumTest") - public void enumTest() { - - String templateGenerr = CommonErrMsg.TemplateGenErr.getMsgTemplate(); - String fitErrMsgTemplate = CommonErrMsg.TemplateFitErr.getMsgTemplate(); - String parserInitErrMsgTemplate = CommonErrMsg.ParserInitErr.getMsgTemplate(); - String parseErrMsgTemplate = CommonErrMsg.ParserParseErr.getMsgTemplate(); - - Assertions.assertEquals("Cannot generate template. :{0}", templateGenerr); - Assertions.assertEquals("Cannot fit input into template: {0}", fitErrMsgTemplate); - - Assertions.assertEquals("Failed to init parser: {0}", parserInitErrMsgTemplate); - Assertions.assertEquals("Failed to parse. {0}", parseErrMsgTemplate); - } -} diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/test/java/org/apache/linkis/cli/core/exception/handler/CommandExceptionHandlerTest.java b/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/test/java/org/apache/linkis/cli/core/exception/handler/CommandExceptionHandlerTest.java deleted file mode 100644 index ee23f0216b..0000000000 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/test/java/org/apache/linkis/cli/core/exception/handler/CommandExceptionHandlerTest.java +++ /dev/null @@ -1,75 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.cli.core.exception.handler; - -import org.apache.linkis.cli.common.exception.error.ErrorLevel; -import org.apache.linkis.cli.common.exception.handler.ExceptionHandler; -import org.apache.linkis.cli.core.exception.CommandException; -import org.apache.linkis.cli.core.exception.error.CommonErrMsg; -import org.apache.linkis.cli.core.interactor.command.TestCmdType; - -import org.junit.jupiter.api.Disabled; -import org.junit.jupiter.api.Test; - -import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; - -public class CommandExceptionHandlerTest { - ExceptionHandler handler = new CommandExceptionHandler(); - - // todo - @Disabled - @Test - public void handle() throws Exception { - CommandException cmdException = - new CommandException( - "CODE-001", - ErrorLevel.ERROR, - CommonErrMsg.TemplateGenErr, - "Failed to generate template."); - assertDoesNotThrow(() -> handler.handle(cmdException)); - - String[] params = {"param1", "param2"}; - CommandException cmdException2 = - new CommandException( - "CODE-001", - ErrorLevel.ERROR, - CommonErrMsg.TemplateGenErr, - params, - "Failed to generate template."); - assertDoesNotThrow(() -> handler.handle(cmdException2)); - - CommandException cmdException3 = - new CommandException( - "CODE-001", - ErrorLevel.ERROR, - CommonErrMsg.TemplateGenErr, - TestCmdType.PRIMARY, - "Failed to generate template."); - assertDoesNotThrow(() -> handler.handle(cmdException3)); - - CommandException cmdException4 = - new CommandException( - "CODE-001", - ErrorLevel.ERROR, - CommonErrMsg.TemplateGenErr, - TestCmdType.PRIMARY, - params, - "Failed to generate template."); - assertDoesNotThrow(() -> handler.handle(cmdException4)); - } -} diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/test/java/org/apache/linkis/cli/core/interactor/command/TestCmdType.java b/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/test/java/org/apache/linkis/cli/core/interactor/command/TestCmdType.java deleted file mode 100644 index a6b05eef2b..0000000000 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/test/java/org/apache/linkis/cli/core/interactor/command/TestCmdType.java +++ /dev/null @@ -1,58 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.cli.core.interactor.command; - -import org.apache.linkis.cli.common.entity.command.CmdType; -import org.apache.linkis.cli.core.constants.TestConstants; - -public enum TestCmdType implements CmdType { - PRIMARY(TestConstants.PRIMARY, 0, TestConstants.PRIMARY_DESC), - SPARK(TestConstants.SPARK, 1, TestConstants.SPARK_DESC); - // TEST(TestConstants.TEST_COMMAND, 0, TestConstants.TEST_DESC); - - private int id; - private String name; - private String desc; - - TestCmdType(String name, int id) { - this.id = id; - this.name = name; - this.desc = null; - } - - TestCmdType(String name, int id, String desc) { - this.id = id; - this.name = name; - this.desc = desc; - } - - @Override - public int getId() { - return this.id; - } - - @Override - public String getName() { - return this.name; - } - - @Override - public String getDesc() { - return this.desc; - } -} diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/test/java/org/apache/linkis/cli/core/interactor/command/fitter/FitterUtilsTest.java b/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/test/java/org/apache/linkis/cli/core/interactor/command/fitter/FitterUtilsTest.java deleted file mode 100644 index 5c51a582fa..0000000000 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/test/java/org/apache/linkis/cli/core/interactor/command/fitter/FitterUtilsTest.java +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.cli.core.interactor.command.fitter; - -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.DisplayName; -import org.junit.jupiter.api.Test; - -public class FitterUtilsTest { - - @Test - @DisplayName("isOptionTest") - public void isOptionTest() { - - String name = "-hadoop"; - boolean option = FitterUtils.isOption(name); - Assertions.assertTrue(option); - } -} diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/test/java/org/apache/linkis/cli/core/interactor/command/fitter/SingleTplFitterTest.java b/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/test/java/org/apache/linkis/cli/core/interactor/command/fitter/SingleTplFitterTest.java deleted file mode 100644 index ef542910dd..0000000000 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/test/java/org/apache/linkis/cli/core/interactor/command/fitter/SingleTplFitterTest.java +++ /dev/null @@ -1,119 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.cli.core.interactor.command.fitter; - -import org.apache.linkis.cli.common.entity.command.CmdTemplate; -import org.apache.linkis.cli.core.interactor.command.template.TestSparkCmdTemplate; - -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; - -import static org.junit.jupiter.api.Assertions.*; - -public class SingleTplFitterTest { - Fitter fitter; - CmdTemplate template; - String[] cmdStr, cmdStr2; - - @BeforeEach - public void before() throws Exception { - cmdStr = - new String[] { - "-u", - "hadoop", - "-pwd", - "1234", - "-c", - "/path/to/user/config", - "--cmd", - "show tables", - "--split", - "\',\'", - "--queue", - "q05", - "--name", - "testApp", - // "--hiveconf", "/path/...", - "--num-executors", - "4", - "--executor-cores", - "4", - "--executor-memory", - "4G", - "--shuffle-partitions", - "200", - "--other", - "--other-spark-config=none", - }; - cmdStr2 = - new String[] { - "-u", - "hadoop", - "-pwd", - "1234", - "-c", - "/path/to/user/config", - "--cmd", - "show tables", - "--split", - "\',\'", - "--queue", - "q05", - "--name", - "testApp", - // "--hiveconf", "/path/...", - "--num-executors", - "4", - "--executor-cores", - "4", - "--executor-memory", - "4G", - "--shuffle-partitions", - "200", - "--other", - "--other-spark-config=none", - "-P", - "key1=value1, key2=value2, key5=\"key3=value3,key4=value4\" " - }; - template = new TestSparkCmdTemplate(); - fitter = new SingleTplFitter(); - } - - @AfterEach - public void after() throws Exception {} - - /** Method: fit(TemplateFitterInput[] inputs) */ - @Test - public void testParseAndFit() throws Exception { - FitterResult[] results = new FitterResult[2]; - results[0] = fitter.fit(cmdStr, template); - results[1] = fitter.fit(cmdStr2, new TestSparkCmdTemplate()); - - assertTrue(results[0].getParsedTemplate() instanceof TestSparkCmdTemplate); - assertEquals( - results[0].getParsedTemplate().getOptionsMap().get("--cmd").getValue(), "show tables"); - assertNotEquals(results[0].getParsedTemplate(), template.getCopy()); - assertNotEquals(results[0].getParsedTemplate().getOptions(), template.getCopy().getOptions()); - assertNotEquals( - results[0].getParsedTemplate().getOptions().get(1), template.getCopy().getOptions().get(1)); - assertEquals( - results[0].getParsedTemplate().getOptions().get(1).getValue(), - template.getCopy().getOptions().get(1).getValue()); - } -} diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/test/java/org/apache/linkis/cli/core/interactor/command/parser/SingleCmdParserTest.java b/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/test/java/org/apache/linkis/cli/core/interactor/command/parser/SingleCmdParserTest.java deleted file mode 100644 index 61b8c72eab..0000000000 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/test/java/org/apache/linkis/cli/core/interactor/command/parser/SingleCmdParserTest.java +++ /dev/null @@ -1,177 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.cli.core.interactor.command.parser; - -import org.apache.linkis.cli.common.entity.command.CmdTemplate; -import org.apache.linkis.cli.common.entity.command.ParamItem; -import org.apache.linkis.cli.core.interactor.command.TestCmdType; -import org.apache.linkis.cli.core.interactor.command.fitter.SingleTplFitter; -import org.apache.linkis.cli.core.interactor.command.parser.result.ParseResult; -import org.apache.linkis.cli.core.interactor.command.template.TestParamMapper; -import org.apache.linkis.cli.core.interactor.command.template.TestSparkCmdTemplate; - -import java.util.Arrays; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.stream.Collectors; - -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; - -import static org.junit.jupiter.api.Assertions.*; - -public class SingleCmdParserTest { - String[] cmdStr; - Map templateMap; - - @BeforeEach - public void before() throws Exception { - cmdStr = - new String[] { - "-u", - "hadoop", - "-pwd", - "1234", - "-c", - "/path/to/user/config", - "--cmd", - "show tables", - "--split", - "\',\'", - "--queue", - "q05", - "--name", - "testApp", - // "--hiveconf", "/path/...", - "--num-executors", - "4", - "--executor-cores", - "4", - "--executor-memory", - "4G", - "--shuffle-partitions", - "200", - "--other", - "--other-spark-config=none", - }; - - TestSparkCmdTemplate template = new TestSparkCmdTemplate(); - templateMap = new HashMap<>(); - templateMap.put(template.getCmdType().getName(), template); - } - - @AfterEach - public void after() throws Exception {} - - /** Method: parse(String[] input) */ - @Test - public void testParse() { - - Parser parser = - new SingleCmdParser() - .setMapper(null) - .setTemplate(templateMap.get(TestCmdType.SPARK.getName())) - .setFitter(new SingleTplFitter()) - .setMapper(new TestParamMapper()); - - ParseResult result = parser.parse(cmdStr); - - assertEquals(result.getParams().getCmdType(), TestCmdType.SPARK); - - Map params = result.getParams().getParamItemMap(); - - assertEquals(params.size(), 21); - - List sortedKeys = params.keySet().stream().sorted().collect(Collectors.toList()); - assertEquals( - sortedKeys, - Arrays.asList( - "converted.args", - "converted.split", - "param.common.command", - "param.common.file", - "param.common.other.kv", - "param.help", - "param.primary.database", - "param.primary.password", - "param.primary.proxy", - "param.primary.proxyUser", - "param.primary.synckey", - "param.primary.user", - "param.primary.user.conf", - "param.spark.executor.cores", - "param.spark.executor.memory", - "param.spark.hiveconf", - "param.spark.name", - "param.spark.num.executors", - "param.spark.runtype", - "param.spark.shuffle.partitions", - "param.yarn.queue")); - assertEquals(result.getRemains().length, 0); - } - - /** Method: parsePrimary(String[] input) */ - @Test - public void testParsePrimary() throws Exception { - // TODO: Test goes here... - /* - try { - Method method = SingleCmdParser.getClass().getMethod("parsePrimary", String[].class); - method.setAccessible(true); - method.invoke(, ); - } catch(NoSuchMethodException e) { - } catch(IllegalAccessException e) { - } catch(InvocationTargetException e) { - } - */ - } - - /** Method: parseSingleSub(String[] remains) */ - @Test - public void testParseSingleSub() throws Exception { - // TODO: Test goes here... - /* - try { - Method method = SingleCmdParser.getClass().getMethod("parseSingleSub", String[].class); - method.setAccessible(true); - method.invoke(, ); - } catch(NoSuchMethodException e) { - } catch(IllegalAccessException e) { - } catch(InvocationTargetException e) { - } - */ - } - - /** Method: standardParse(String identifier, String[] args, CmdTemplate templateOri) */ - @Test - public void testStandardParse() throws Exception { - // TODO: Test goes here... - /* - try { - Method method = SingleCmdParser.getClass().getMethod("standardParse", String.class, String[].class, CmdTemplate.class); - method.setAccessible(true); - method.invoke(, ); - } catch(NoSuchMethodException e) { - } catch(IllegalAccessException e) { - } catch(InvocationTargetException e) { - } - */ - } -} diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/test/java/org/apache/linkis/cli/core/interactor/command/template/TestParamMapper.java b/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/test/java/org/apache/linkis/cli/core/interactor/command/template/TestParamMapper.java deleted file mode 100644 index 25f44d56c7..0000000000 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/test/java/org/apache/linkis/cli/core/interactor/command/template/TestParamMapper.java +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.cli.core.interactor.command.template; - -import org.apache.linkis.cli.core.constants.TestConstants; -import org.apache.linkis.cli.core.interactor.command.parser.transformer.ParamKeyMapper; - -public class TestParamMapper extends ParamKeyMapper { - - @Override - public void initMapperRules() { - super.updateMapping(TestConstants.SPARK, "converted.cmd"); - super.updateMapping(TestConstants.PARAM_COMMON_ARGS, "converted.args"); - super.updateMapping(TestConstants.PARAM_COMMON_SPLIT, "converted.split"); - // super.updateMapping("key1", "spark.cmd"); //should throw exception - // super.updateMapping("TestConstants.PARAM_SPARK_CMD", "spark.cmd"); - // super.updateMapping("TestConstants.PARAM_SPARK_CMD", "spark.cmd"); - - } -} diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/test/java/org/apache/linkis/cli/core/interactor/command/template/TestSparkCmdTemplate.java b/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/test/java/org/apache/linkis/cli/core/interactor/command/template/TestSparkCmdTemplate.java deleted file mode 100644 index 4795f11c9c..0000000000 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/test/java/org/apache/linkis/cli/core/interactor/command/template/TestSparkCmdTemplate.java +++ /dev/null @@ -1,268 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.cli.core.interactor.command.template; - -import org.apache.linkis.cli.common.exception.LinkisClientRuntimeException; -import org.apache.linkis.cli.core.constants.TestConstants; -import org.apache.linkis.cli.core.exception.CommandException; -import org.apache.linkis.cli.core.interactor.command.TestCmdType; -import org.apache.linkis.cli.core.interactor.command.template.option.StdOption; - -import org.apache.commons.lang3.StringUtils; - -import java.util.HashMap; -import java.util.Map; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public class TestSparkCmdTemplate extends AbstractCmdTemplate { - protected StdOption databaseOp = - option( - null, - TestConstants.PARAM_DB, - new String[] {"-d", "--database"}, - "specify database", - true, - ""); - protected StdOption proxyOp = - option( - null, - TestConstants.PARAM_PROXY, - new String[] {"-x", "--proxy"}, - "specify proxy url", - true, - ""); - protected StdOption userOp = - option( - null, TestConstants.PARAM_USER, new String[] {"-u", "--user"}, "specify user", true, ""); - protected StdOption confOp = - option( - null, - TestConstants.PARAM_USR_CONF, - new String[] {"-c", "--conf"}, - "specify configuration from property file", - true, - ""); - private Logger logger = LoggerFactory.getLogger(TestSparkCmdTemplate.class); - private StdOption passwordOp = - option( - null, - TestConstants.PARAM_PASSWORD, - new String[] {"-pwd", "--passwd"}, - "specify user password", - true, - ""); - private StdOption syncOp = - option( - null, - TestConstants.PARAM_SYNC_KEY, - new String[] {"-sk", "--synckey"}, - "specify sync key", - true, - ""); - private StdOption proxyUserOp = - option( - null, - TestConstants.PARAM_PROXY_USER, - new String[] {"-pu", "--proxy-user"}, - "specify proxy user", - true, - ""); - - private StdOption helpOp = - option(null, TestConstants.PARAM_HELP, new String[] {"-h", "--help"}, "help info", true, ""); - - private StdOption filePara = - option( - null, - TestConstants.PARAM_COMMON_FILE, - new String[] {"--file", "-f"}, - "Spark SQL File to Execute!", - true, - ""); - - private StdOption commandPara = - option( - null, - TestConstants.PARAM_COMMON_CMD, - new String[] {"--cmd"}, - "Spark SQL Command to Execute!", - true, - ""); - - private StdOption argsPara = - option( - null, - TestConstants.PARAM_COMMON_ARGS, - new String[] {"--args", "-a"}, - "Set command args, k-v pairs delimited by comma, e.g. key1=value1,key2=value2,...", - true, - ""); - - private StdOption splitPara = - option( - null, - TestConstants.PARAM_COMMON_SPLIT, - new String[] {"--split", "-s"}, - "specify the split character string", - true, - ","); - - private StdOption queuePara = - option( - null, - TestConstants.PARAM_YARN_QUEUE, - new String[] {"--queue", "-q"}, - "specify the queue", - true, - "default"); - - private StdOption namePara = - option( - null, - TestConstants.PARAM_SPARK_NAME, - new String[] {"--name", "-n"}, - "specify the application name. WARNING:this option is deprecated. Linkis does not support this variable", - true, - ""); - - private StdOption> hiveconfPara = - option( - null, - TestConstants.PARAM_SPARK_HIVECONF, - new String[] {"--hiveconf", "-hc"}, - "specify the hiveconf setting,e.g. hive.cli.print.header=false", - true, - new HashMap<>()); - - private StdOption nePara = - option( - null, - TestConstants.PARAM_SPARK_NUM_EXECUTORS, - new String[] {"--num-executors", "-ne"}, - "specify the spark application container", - true, - 3); - - private StdOption ecPara = - option( - null, - TestConstants.PARAM_SPARK_EXECUTOR_CORES, - new String[] {"--executor-cores", "-ec"}, - "specify the spark application container vcores(less than queue's max vcores)", - true, - 2); - - private StdOption emPara = - option( - null, - TestConstants.PARAM_SPARK_EXECUTOR_MEMORY, - new String[] {"--executor-memory", "-em"}, - "specify the spark application executor's memory, 1.5G-2G/vcore", - true, - "4G"); - - private StdOption spPara = - option( - null, - TestConstants.PARAM_SPARK_SHUFFLE_PARTITIONS, - new String[] {"--shuffle-partitions", "-sp"}, - "specify the spark.sql.shuffle.partitions", - true, - 200); - - private StdOption otherPara = - option( - null, - TestConstants.PARAM_COMMON_OTHER_KV, - new String[] {"--other"}, - "specify the other parameters", - true, - ""); - - private StdOption runTypePara = - option( - null, - TestConstants.PARAM_SPARK_RUNTYPE, - new String[] {"--runtype"}, - "specify the runtype parameters: sql pyspark scala", - true, - "sql"); - - public TestSparkCmdTemplate() { - super(TestCmdType.SPARK); - } - - @Override - public void checkParams() throws CommandException {} - - @Override - protected Object clone() throws CloneNotSupportedException { - return super.clone(); - } - - @Override - public TestSparkCmdTemplate getCopy() { - return (TestSparkCmdTemplate) super.getCopy(); - } - - public void prepare() throws LinkisClientRuntimeException {} - - private Map parseOtherMap(String otherStr) { - Map otherMap = new HashMap<>(); - /** - * split by space-chara, but if space char is within \"\", then do not split. e.g. "a b c" will - * not be splited - */ - otherStr = otherStr.replace("\'", "\""); - String[] arr = otherStr.trim().split("\\s(?=(?:[^\"]*\"[^\"]*\")*[^\"]*$)", -1); - for (int i = 0; i < arr.length; i++) { - String candidate = arr[i]; - - if (StringUtils.equalsIgnoreCase(candidate, "--conf") && i < arr.length - 1) { - // '--conf' and has next - i++; // move to next - candidate = arr[i]; - int idx = StringUtils.indexOf(candidate, "="); - if (idx != -1) { - String key = StringUtils.substring(candidate, 0, idx).trim(); - String value = StringUtils.substring(candidate, idx + 1).trim(); - otherMap.put(key, value); - } - } else { - // without '--conf' - int idx = StringUtils.indexOf(candidate, " "); - if (idx != -1) { - // e.g. '--driver-memory 2G' - String key = StringUtils.substring(candidate, 0, idx).trim(); - String value = StringUtils.substring(candidate, idx + 1).trim(); - otherMap.put(key, value); - } else if (i < arr.length - 1) { - // e.g. --driver-memory 2G - String key = candidate.trim(); - i++; // move to next - candidate = arr[i]; - String value = candidate.trim(); - otherMap.put(key, value); - } - } - } - return otherMap; - } -} diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/test/java/org/apache/linkis/cli/core/interactor/properties/StdPropsLoaderTest.java b/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/test/java/org/apache/linkis/cli/core/interactor/properties/StdPropsLoaderTest.java deleted file mode 100644 index 9bef89b3f3..0000000000 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/test/java/org/apache/linkis/cli/core/interactor/properties/StdPropsLoaderTest.java +++ /dev/null @@ -1,119 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.cli.core.interactor.properties; - -import org.apache.linkis.cli.common.entity.properties.ClientProperties; -import org.apache.linkis.cli.core.interactor.properties.reader.PropertiesReader; - -import java.util.Arrays; -import java.util.List; -import java.util.stream.Collectors; - -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; - -import static org.junit.jupiter.api.Assertions.assertEquals; - -public class StdPropsLoaderTest { - PropertiesLoader loader; - - @BeforeEach - public void before() throws Exception { - System.setProperty("conf.root", "src/test/resources/conf/"); - System.setProperty("conf.file", "linkis-cli.properties"); - String configPath = System.getProperty("conf.root"); - String defaultConfFileName = System.getProperty("conf.file"); - /* - default config, -Dconf.root & -Dconf.file specifies config path - */ - List readersList = - new PropsFilesScanner().getPropsReaders(configPath); // +1 user config - /* - user defined config - */ - // load all config files - loader = - new StdPropsLoader() - .addPropertiesReaders(readersList.toArray(new PropertiesReader[readersList.size()])); - } - - @AfterEach - public void after() throws Exception {} - - /** Method: setPropertiesReaders(PropertiesReader[] readers) */ - @Test - public void testSetPropertiesReaders() throws Exception { - // TODO: Test goes here... - loader.setPropertiesReaders(new PropertiesReader[0]); - } - - /** Method: getAllReadersAsMap() */ - @Test - public void testGetAllReadersAsMap() throws Exception { - // TODO: Test goes here... - } - - /** Method: addPropertiesReader(PropertiesReader reader) */ - @Test - public void testAddPropertiesReader() throws Exception { - // TODO: Test goes here... - } - - /** Method: addPropertiesReaders(PropertiesReader[] readers) */ - @Test - public void testAddPropertiesReaders() throws Exception { - // TODO: Test goes here... - } - - /** Method: getPropertiesReader(String identifier) */ - @Test - public void testGetPropertiesReader() throws Exception { - // TODO: Test goes here... - } - - /** Method: removePropertiesReader(String identifier) */ - @Test - public void testRemovePropertiesReader() throws Exception { - // TODO: Test goes here... - } - - /** Method: loadProperties() */ - @Test - public void testLoadProperties() throws Exception { - ClientProperties[] loaderResult = loader.loadProperties(); - - List properties = - Arrays.stream(loaderResult) - .sorted((p1, p2) -> p1.size() - p2.size()) - .collect(Collectors.toList()); - - assertEquals(2, properties.size()); - assertEquals(properties.get(0).getPropsId(), "user.properties"); - assertEquals(properties.get(0).size(), 4); - - assertEquals(properties.get(1).getPropsId(), "linkis-cli.properties"); - assertEquals(properties.get(1).size(), 8); - } - - /** Method: checkInit() */ - @Test - public void testCheckInit() throws Exception { - // TODO: Test goes here... - } -} diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/test/java/org/apache/linkis/cli/core/interactor/var/StdVarAccessTest.java b/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/test/java/org/apache/linkis/cli/core/interactor/var/StdVarAccessTest.java deleted file mode 100644 index 497a539dd5..0000000000 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/test/java/org/apache/linkis/cli/core/interactor/var/StdVarAccessTest.java +++ /dev/null @@ -1,298 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.cli.core.interactor.var; - -import org.apache.linkis.cli.common.entity.command.CmdTemplate; -import org.apache.linkis.cli.common.entity.properties.ClientProperties; -import org.apache.linkis.cli.common.entity.var.VarAccess; -import org.apache.linkis.cli.core.constants.TestConstants; -import org.apache.linkis.cli.core.interactor.command.TestCmdType; -import org.apache.linkis.cli.core.interactor.command.fitter.SingleTplFitter; -import org.apache.linkis.cli.core.interactor.command.parser.Parser; -import org.apache.linkis.cli.core.interactor.command.parser.SingleCmdParser; -import org.apache.linkis.cli.core.interactor.command.parser.result.ParseResult; -import org.apache.linkis.cli.core.interactor.command.template.TestSparkCmdTemplate; -import org.apache.linkis.cli.core.interactor.properties.PropertiesLoader; -import org.apache.linkis.cli.core.interactor.properties.PropsFilesScanner; -import org.apache.linkis.cli.core.interactor.properties.StdPropsLoader; -import org.apache.linkis.cli.core.interactor.properties.reader.PropertiesReader; -import org.apache.linkis.cli.core.interactor.properties.reader.PropsFileReader; -import org.apache.linkis.cli.core.interactor.validate.ParsedTplValidator; - -import org.apache.commons.lang3.StringUtils; - -import java.util.Arrays; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertTrue; - -public class StdVarAccessTest { - String[] cmdStr; - VarAccess stdVarAccess; - - @BeforeEach - public void before() throws Exception { - cmdStr = - new String[] { - "-u", - "hadoop", - "-pwd", - "1234", - "-c", - "src/test/resources/conf/user.properties", - "--cmd", - "show tables", - "--split", - "\',\'", - "--queue", - "q05", - "--name", - "testApp", - // "--hiveconf", "/path/...", - "--num-executors", - "4", - "--executor-cores", - "4", - "--executor-memory", - "4G", - "--shuffle-partitions", - "200", - "--other", - "--other-spark-config=none", - }; - - TestSparkCmdTemplate template = new TestSparkCmdTemplate(); - Map templateMap = new HashMap<>(); - templateMap.put(template.getCmdType().getName(), template); - - Parser parser = - new SingleCmdParser() - .setMapper(null) - .setTemplate(templateMap.get(TestCmdType.SPARK.getName())) - .setFitter(new SingleTplFitter()); - - ParseResult result = parser.parse(cmdStr); - - ParsedTplValidator parsedTplValidator = new ParsedTplValidator(); - parsedTplValidator.doValidation(result.getParsedTemplate()); - - System.setProperty("conf.root", "src/test/resources/conf/"); - System.setProperty("conf.file", "linkis-cli.properties"); - String configPath = System.getProperty("conf.root"); - String defaultConfFileName = System.getProperty("conf.file"); - /* - default config, -Dconf.root & -Dconf.file specifies config path - */ - List readersList = - new PropsFilesScanner().getPropsReaders(configPath); // +1 user config - /* - user defined config - */ - String userConfPath = - (String) result.getParams().getParamItemMap().get(TestConstants.PARAM_USR_CONF).getValue(); - if (StringUtils.isNotBlank(userConfPath)) { - PropertiesReader reader = new PropsFileReader(); - reader.setPropsId("user.conf"); - reader.setPropsPath(userConfPath); - readersList.add(reader); - } else { - } - // load all config files - PropertiesLoader loader = - new StdPropsLoader() - .addPropertiesReaders(readersList.toArray(new PropertiesReader[readersList.size()])); - ClientProperties[] loaderResult = loader.loadProperties(); - Map propertiesMap = new HashMap<>(); - for (ClientProperties properties : loaderResult) { - propertiesMap.put(properties.getPropsId(), properties); - } - - stdVarAccess = - new StdVarAccess() - .setCmdParams(result.getParams()) - .setUserConf(propertiesMap.get("user.conf")) - .setDefaultConf(propertiesMap.get(defaultConfFileName)) - .init(); - } - - @AfterEach - public void after() throws Exception {} - - /** Method: setPrimaryParam(Params primaryParam) */ - @Test - public void testSetPrimaryParam() throws Exception { - // TODO: Test goes here... - } - - /** Method: getPrimaryParam(String identifier) */ - @Test - public void testGetPrimaryParam() throws Exception { - // TODO: Test goes here... - } - - /** Method: setCmdParams(Params subParam) */ - @Test - public void testSetSubParam() throws Exception { - // TODO: Test goes here... - } - - /** Method: getSubParam(String identifier) */ - @Test - public void testGetSubParam() throws Exception { - // TODO: Test goes here... - } - - /** Method: setUserConf(ClientProperties userConf) */ - @Test - public void testSetUserConf() throws Exception { - // TODO: Test goes here... - } - - /** Method: getUserConf(String identifier) */ - @Test - public void testGetUserConf() throws Exception { - // TODO: Test goes here... - } - - /** Method: setDefaultConf(ClientProperties defaultConf) */ - @Test - public void testSetDefaultConf() throws Exception { - // TODO: Test goes here... - } - - /** Method: getDefaultConf(String identifier) */ - @Test - public void testGetDefaultConf() throws Exception { - // TODO: Test goes here... - } - - /** Method: checkInit() */ - @Test - public void testCheckInit() throws Exception { - // TODO: Test goes here... - } - - /** Method: getVar(Class clazz, String key) */ - @Test - public void testGetVar() throws Exception { - System.out.println(stdVarAccess.getVar(String.class, TestConstants.PARAM_COMMON_CMD)); - assertEquals(stdVarAccess.getVar(String.class, TestConstants.PARAM_COMMON_CMD), "show tables"); - System.out.println(stdVarAccess.getVar(String.class, "user.props")); - assertEquals(stdVarAccess.getVar(String.class, "wds.linkis.client.not.exist"), null); - System.out.println( - stdVarAccess.getVar( - Integer.class, TestConstants.PARAM_SPARK_EXECUTOR_CORES)); // see if priority works - assertEquals( - (long) stdVarAccess.getVar(Integer.class, TestConstants.PARAM_SPARK_EXECUTOR_CORES), 4); - assertEquals((long) stdVarAccess.getVar(Integer.class, "conf.prop.integer"), 9); - assertEquals(stdVarAccess.getVar(String.class, "conf.prop.string"), "str"); - assertEquals( - stdVarAccess.getVar(String.class, "wds.linkis.client.param.conf.spark.executor.memory"), - "11111G"); - - System.out.println(stdVarAccess.getAllVarKeys().length); - System.out.println(Arrays.toString(stdVarAccess.getAllVarKeys())); - assertTrue(stdVarAccess.getAllVarKeys().length != 0); - } - - /** Method: getVarOrDefault(Class clazz, String key, T defaultValue) */ - @Test - public void testGetVarOrDefault() throws Exception { - // TODO: Test goes here... - } - - /** Method: getAllVarKeys() */ - @Test - public void testGetAllVarKeys() throws Exception { - // TODO: Test goes here... - } - - /** Method: getVarFromParam(Class clazz, String key, Params param) */ - @Test - public void testGetVarFromParam() throws Exception { - // TODO: Test goes here... - /* - try { - Method method = StdVarAccess.getClass().getMethod("getVarFromParam", Class.class, String.class, Params.class); - method.setAccessible(true); - method.invoke(, ); - } catch(NoSuchMethodException e) { - } catch(IllegalAccessException e) { - } catch(InvocationTargetException e) { - } - */ - } - - /** Method: getDefaultVarFromParam(Class clazz, String key, Params param) */ - @Test - public void testGetDefaultVarFromParam() throws Exception { - // TODO: Test goes here... - /* - try { - Method method = StdVarAccess.getClass().getMethod("getDefaultVarFromParam", Class.class, String.class, Params.class); - method.setAccessible(true); - method.invoke(, ); - } catch(NoSuchMethodException e) { - } catch(IllegalAccessException e) { - } catch(InvocationTargetException e) { - } - */ - } - - /** Method: getVarFromCfg(Class clazz, String key, ClientProperties conf) */ - @Test - public void testGetVarFromCfg() throws Exception { - // TODO: Test goes here... - /* - try { - Method method = StdVarAccess.getClass().getMethod("getVarFromCfg", Class.class, String.class, ClientProperties.class); - method.setAccessible(true); - method.invoke(, ); - } catch(NoSuchMethodException e) { - } catch(IllegalAccessException e) { - } catch(InvocationTargetException e) { - } - */ - } - - /** - * Method: getVarFromCfgGivenConverter(String key, ClientProperties conf, - * AbstractStringConverter converter) - */ - @Test - public void testGetVarFromCfgGivenConverter() throws Exception { - // TODO: Test goes here... - /* - try { - Method method = StdVarAccess.getClass().getMethod("getVarFromCfgGivenConverter", String.class, ClientProperties.class, AbstractStringConverter.class); - method.setAccessible(true); - method.invoke(, ); - } catch(NoSuchMethodException e) { - } catch(IllegalAccessException e) { - } catch(InvocationTargetException e) { - } - */ - } -} diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/test/java/org/apache/linkis/cli/core/present/PresentModeImplTest.java b/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/test/java/org/apache/linkis/cli/core/present/PresentModeImplTest.java deleted file mode 100644 index 8f776bef81..0000000000 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/test/java/org/apache/linkis/cli/core/present/PresentModeImplTest.java +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.cli.core.present; - -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.DisplayName; -import org.junit.jupiter.api.Test; - -public class PresentModeImplTest { - - @Test - @DisplayName("enumTest") - public void enumTest() { - - String stdoutName = PresentModeImpl.STDOUT.getName(); - String textFileName = PresentModeImpl.TEXT_FILE.getName(); - - Assertions.assertEquals("STDOUT", stdoutName); - Assertions.assertEquals("TEXT_FILE", textFileName); - } -} diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/test/java/org/apache/linkis/cli/core/utils/TestUtils.java b/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/test/java/org/apache/linkis/cli/core/utils/TestUtils.java deleted file mode 100644 index 61cdd6aa7f..0000000000 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/test/java/org/apache/linkis/cli/core/utils/TestUtils.java +++ /dev/null @@ -1,73 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.cli.core.utils; - -import org.apache.commons.io.FileUtils; -import org.apache.commons.lang3.StringUtils; - -import java.io.File; -import java.util.HashMap; -import java.util.Map; - -public class TestUtils { - private static String replaceParas(String context, Map m) { - if (context == null) { - return null; - } - for (Map.Entry entry : m.entrySet()) { - if (entry.getKey() == null || entry.getValue() == null) { - continue; - } - String key = "[#" + entry.getKey().trim() + "]"; - String value = entry.getValue().trim(); - - context = StringUtils.replace(context, key, value); - } - return context; - } - - public static Map parseArgMap(String str, String separator) { - - Map argsProps = new HashMap<>(); - String[] args = StringUtils.splitByWholeSeparator(str, separator); - - for (String arg : args) { - int index = arg.indexOf("="); - if (index != -1) { - argsProps.put(arg.substring(0, index).trim(), arg.substring(index + 1).trim()); - } - } - return argsProps; - } - - public static String readShellFileAndReplaceParas(String filename, String argsStr, String split) - throws Exception { - - String fileContent; - - File inputFile = new File(filename); - - fileContent = FileUtils.readFileToString(inputFile); - - Map argsMap = parseArgMap(argsStr, split); - - fileContent = replaceParas(fileContent, argsMap); - - return fileContent; - } -} diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/test/resources/conf/linkis-cli.properties b/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/test/resources/conf/linkis-cli.properties deleted file mode 100644 index e0f85cf3e5..0000000000 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/test/resources/conf/linkis-cli.properties +++ /dev/null @@ -1,34 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# http://www.apache.org/licenses/LICENSE-2.0 -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# -wds.linkis.client.common.creator=IDE3 -wds.linkis.client.common.gatewayUrl=http://127.0.0.1:8088 -#wds.linkis.client.common.authStrategy -#wds.linki.clients.common.tokenKey -#wds.linkis.client.common.tokenValue -# -#wds.linkis.client.common.submitUser -#wds.linkis.client.common.submitPassword -#wds.linkis.client.common.proxyUser -wds.linkis.client.param.conf.wds.linkis.yarnqueue=q02 -wds.linkis.client.param.conf.yarnqueue.cores.max=233 -wds.linkis.client.param.conf.yarnqueue.memory.max=233G -wds.linkis.client.param.conf.spark.executor.instances=9 -wds.linkis.client.param.conf.spark.executor.cores=9 -wds.linkis.client.param.conf.spark.executor.memory=9 -#wds.linkis.client.label="key1=val1,key2=val2" -#wds.linkis.client.param.conf="key1=val1,key2=val2" -#wds.linkis.client.param.var="key1=val1,key2=val2" -#wds.linkis.client.source="key1=val1,key2=val2" \ No newline at end of file diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/test/resources/conf/user.properties b/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/test/resources/conf/user.properties deleted file mode 100644 index faa650fc5c..0000000000 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/test/resources/conf/user.properties +++ /dev/null @@ -1,19 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# http://www.apache.org/licenses/LICENSE-2.0 -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# -user.props=xxxxxxxxxxxxx -conf.prop.integer=9 -conf.prop.string=str -wds.linkis.client.param.conf.spark.executor.memory=11111G \ No newline at end of file diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/test/resources/log4j2.xml b/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/test/resources/log4j2.xml deleted file mode 100644 index c744678876..0000000000 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/test/resources/log4j2.xml +++ /dev/null @@ -1,50 +0,0 @@ - - - - - - ${sys:log.path} - ${sys:log.file} - - - - - - - - - - - - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/linkis-computation-governance/linkis-client/linkis-cli/pom.xml b/linkis-computation-governance/linkis-client/linkis-cli/pom.xml index e9efc432d5..76723c4adb 100644 --- a/linkis-computation-governance/linkis-client/linkis-cli/pom.xml +++ b/linkis-computation-governance/linkis-client/linkis-cli/pom.xml @@ -25,15 +25,24 @@ ../../../pom.xml linkis-cli - pom - - - linkis-cli-common - linkis-cli-core - linkis-cli-application - + jar + + org.apache.linkis + linkis-computation-client + ${project.version} + + + org.apache.linkis + linkis-gateway-httpclient-support + ${project.version} + + + org.reflections + reflections + ${reflections.version} + org.apache.linkis linkis-common @@ -42,17 +51,53 @@ org.apache.commons commons-lang3 + ${commons-lang3.version} com.google.code.gson gson ${gson.version} - ${project.artifactId}-${project.version} + + + true + ${basedir}/src/main/resources + + + + + org.apache.maven.plugins + maven-assembly-plugin + false + + false + out + false + false + + /src/main/assembly/distribution.xml + + + + + make-assembly + + single + + package + + + /src/main/assembly/distribution.xml + + + + + + diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/assembly/distribution.xml b/linkis-computation-governance/linkis-client/linkis-cli/src/main/assembly/distribution.xml similarity index 98% rename from linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/assembly/distribution.xml rename to linkis-computation-governance/linkis-client/linkis-cli/src/main/assembly/distribution.xml index a1cf04696a..24463e39af 100644 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/assembly/distribution.xml +++ b/linkis-computation-governance/linkis-client/linkis-cli/src/main/assembly/distribution.xml @@ -16,7 +16,7 @@ ~ limitations under the License. --> - linkis-cli diff --git a/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/CtxBuilder.java b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/CtxBuilder.java new file mode 100644 index 0000000000..cfa57d4e26 --- /dev/null +++ b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/CtxBuilder.java @@ -0,0 +1,185 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.cli.application; + +import org.apache.linkis.cli.application.constants.CliConstants; +import org.apache.linkis.cli.application.constants.CliKeys; +import org.apache.linkis.cli.application.entity.command.CmdTemplate; +import org.apache.linkis.cli.application.entity.command.Params; +import org.apache.linkis.cli.application.entity.context.CliCtx; +import org.apache.linkis.cli.application.entity.var.VarAccess; +import org.apache.linkis.cli.application.exception.LinkisClientRuntimeException; +import org.apache.linkis.cli.application.exception.PropsException; +import org.apache.linkis.cli.application.exception.error.CommonErrMsg; +import org.apache.linkis.cli.application.exception.error.ErrorLevel; +import org.apache.linkis.cli.application.interactor.command.CliCmdType; +import org.apache.linkis.cli.application.interactor.command.CmdTemplateFactory; +import org.apache.linkis.cli.application.interactor.command.fitter.SingleTplFitter; +import org.apache.linkis.cli.application.interactor.command.parser.Parser; +import org.apache.linkis.cli.application.interactor.command.parser.SingleCmdParser; +import org.apache.linkis.cli.application.interactor.command.parser.result.ParseResult; +import org.apache.linkis.cli.application.interactor.context.CliCtxImpl; +import org.apache.linkis.cli.application.interactor.properties.ClientProperties; +import org.apache.linkis.cli.application.interactor.properties.PropertiesLoader; +import org.apache.linkis.cli.application.interactor.properties.PropsFilesScanner; +import org.apache.linkis.cli.application.interactor.properties.reader.PropertiesReader; +import org.apache.linkis.cli.application.interactor.properties.reader.PropsFileReader; +import org.apache.linkis.cli.application.interactor.validate.ParsedTplValidator; +import org.apache.linkis.cli.application.interactor.var.VarAccessImpl; +import org.apache.linkis.cli.application.utils.CliUtils; +import org.apache.linkis.cli.application.utils.LoggerManager; + +import org.apache.commons.lang3.StringUtils; + +import java.io.*; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Properties; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class CtxBuilder { + private static Logger logger = LoggerFactory.getLogger(CtxBuilder.class); + + /** generate Templates load env variables TODO: load version info */ + public static CliCtx buildCtx(String[] args) throws LinkisClientRuntimeException { + /* + user input + */ + CmdTemplate template = CmdTemplateFactory.getTemplateCopy(CliCmdType.UNIVERSAL); + Parser parser = + new SingleCmdParser() + .setMapper(null) + .setTemplate(template) + .setFitter(new SingleTplFitter()); + + ParseResult result = parser.parse(args); + + ParsedTplValidator parsedTplValidator = new ParsedTplValidator(); + + parsedTplValidator.doValidation(result.getParsedTemplate()); + + Params params = result.getParams(); + logger.debug("==========params============\n" + CliUtils.GSON.toJson(params)); + + /* + VarAccess for sys_prop, sys_env + */ + + Map propertiesMap = new HashMap<>(); + + LoggerManager.getInformationLogger() + .info( + "LogFile path: " + + System.getProperty(CliKeys.LOG_PATH_KEY) + + "/" + + System.getProperty(CliKeys.LOG_FILE_KEY)); + /* + default config, -Dconf.root & -Dconf.file specifies config path + */ + // scan config files given root path + String configPath = System.getProperty(CliKeys.CLIENT_CONFIG_ROOT_KEY); + String defaultConfFileName = + System.getProperty(CliKeys.DEFAULT_CONFIG_FILE_NAME_KEY, CliConstants.DEFAULT_CONFIG_NAME); + + if (StringUtils.isBlank(configPath)) { + throw new PropsException( + "PRP0007", + ErrorLevel.ERROR, + CommonErrMsg.PropsLoaderErr, + "configuration root path specified by env variable: " + + CliKeys.CLIENT_CONFIG_ROOT_KEY + + " is empty."); + } + + List readersList = + new PropsFilesScanner().getPropsReaders(configPath); // +1 user config + /* + user defined config + */ + String userConfPath = null; + if (params.containsParam(CliKeys.LINKIS_CLIENT_USER_CONFIG)) { + userConfPath = + (String) params.getParamItemMap().get(CliKeys.LINKIS_CLIENT_USER_CONFIG).getValue(); + } + if (StringUtils.isNotBlank(userConfPath)) { + PropertiesReader reader = + new PropsFileReader() + .setPropsId(CliKeys.LINKIS_CLIENT_USER_CONFIG) + .setPropsPath(userConfPath); + readersList.add(reader); + } else { + LoggerManager.getInformationLogger() + .info("User does not provide usr-configuration file. Will use default config"); + } + /* + load properties + */ + PropertiesLoader loader = + new PropertiesLoader() + .addPropertiesReaders(readersList.toArray(new PropertiesReader[readersList.size()])); + ClientProperties[] loaderResult = loader.loadProperties(); + for (ClientProperties properties : loaderResult) { + if (StringUtils.equals(properties.getPropsId(), CliKeys.LINKIS_CLIENT_USER_CONFIG)) { + for (Map.Entry prop : properties.entrySet()) { + if (StringUtils.startsWith( + (String) prop.getKey(), CliKeys.LINKIS_CLIENT_NONCUSTOMIZABLE)) { + throw new PropsException( + "PRP0007", + ErrorLevel.ERROR, + CommonErrMsg.PropsLoaderErr, + "User cannot specify non-customizable configuration: " + prop.getKey()); + } + } + } + propertiesMap.put(properties.getPropsId(), properties); + } + + /* + VarAccess for cmd, config + */ + VarAccess varAccess = + new VarAccessImpl() + .setCmdParams(params) + .setUserConf(propertiesMap.get(CliKeys.LINKIS_CLIENT_USER_CONFIG)) + .setDefaultConf(propertiesMap.get(defaultConfFileName)) + .init(); + logger.info("==========std_var============\n" + CliUtils.GSON.toJson(varAccess)); + + Properties props = new Properties(); + try (InputStream inputStream = + CtxBuilder.class.getClassLoader().getResourceAsStream("version.properties")) { + try (InputStreamReader reader = new InputStreamReader(inputStream)) { + try (BufferedReader bufferedReader = new BufferedReader(reader)) { + props.load(bufferedReader); + } + } + } catch (Exception e) { + logger.warn("Failed to load version info", e); + } + + String verion = props.getProperty(CliKeys.VERSION); + + Map extraMap = new HashMap<>(); + extraMap.put(CliKeys.VERSION, verion); + + return new CliCtxImpl(params.getCmdType(), template, varAccess, extraMap); + } +} diff --git a/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/LinkisClientApplication.java b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/LinkisClientApplication.java new file mode 100644 index 0000000000..9a2284c3a0 --- /dev/null +++ b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/LinkisClientApplication.java @@ -0,0 +1,189 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.cli.application; + +import org.apache.linkis.cli.application.constants.CliConstants; +import org.apache.linkis.cli.application.constants.CliKeys; +import org.apache.linkis.cli.application.entity.command.CmdTemplate; +import org.apache.linkis.cli.application.entity.context.CliCtx; +import org.apache.linkis.cli.application.entity.job.Job; +import org.apache.linkis.cli.application.entity.job.JobResult; +import org.apache.linkis.cli.application.exception.CommandException; +import org.apache.linkis.cli.application.interactor.command.CmdTemplateFactory; +import org.apache.linkis.cli.application.interactor.command.template.UniversalCmdTemplate; +import org.apache.linkis.cli.application.interactor.job.help.HelpJob; +import org.apache.linkis.cli.application.interactor.job.interactive.InteractiveJob; +import org.apache.linkis.cli.application.interactor.job.jobcmd.JobCmdJob; +import org.apache.linkis.cli.application.interactor.job.once.LinkisOnceJob; +import org.apache.linkis.cli.application.interactor.job.version.VersionJob; +import org.apache.linkis.cli.application.operator.OperManager; +import org.apache.linkis.cli.application.operator.once.OnceOperBuilder; +import org.apache.linkis.cli.application.operator.ujes.LinkisOperBuilder; +import org.apache.linkis.cli.application.present.HelpPresenter; +import org.apache.linkis.cli.application.present.model.HelpInfoModel; +import org.apache.linkis.cli.application.utils.LoggerManager; +import org.apache.linkis.cli.application.utils.SchedulerManager; + +import org.apache.commons.lang3.exception.ExceptionUtils; + +import java.util.HashMap; +import java.util.Map; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class LinkisClientApplication { + private static Logger logger = LoggerFactory.getLogger(LinkisClientApplication.class); + + private static boolean showHelp = false; + + public static void main(String[] args) { + /* + generate template + */ + CmdTemplateFactory.register(new UniversalCmdTemplate()); + + /* + build ctx + */ + CliCtx ctx = null; + try { + ctx = CtxBuilder.buildCtx(args); + } catch (CommandException e) { + CmdTemplate template = CmdTemplateFactory.getTemplateOri(e.getCmdType()); + if (template != null) { + HelpInfoModel model = new HelpInfoModel(); + model.buildModel(ctx.getTemplate()); + new HelpPresenter().present(model); + } + LoggerManager.getInformationLogger().error("Failed to build CliCtx", e); + System.exit(-1); + } + + /* + prepare oper + */ + OperManager.register(CliKeys.Linkis_OPER, new LinkisOperBuilder()); + OperManager.register(CliKeys.LINKIS_ONCE, new OnceOperBuilder()); + + /* + run job + */ + Job job; + if (isVersionCmd(ctx)) { + job = new VersionJob(); + } else if (isHelp(ctx)) { + job = new HelpJob(); + } else if (isJobCmd(ctx)) { + job = new JobCmdJob(); + } else if (isOnceCmd(ctx)) { + job = new LinkisOnceJob(); + } else { + job = new InteractiveJob(); + } + job.build(ctx); + JobResult result; + try { + Runtime.getRuntime() + .addShutdownHook( + new Thread( + () -> { + if (job != null) { + job.onDestroy(); + } + })); + result = job.run(); + } catch (Exception e) { + logger.error("Failed to execute job", e); + result = + new JobResult() { + @Override + public Boolean isSuccess() { + return false; + } + + @Override + public String getMessage() { + return "Failed to execute job" + ExceptionUtils.getStackTrace(e); + } + + @Override + public Map getExtraMessage() { + return new HashMap<>(); + } + }; + } + + /* + process result + */ + printIndicator(result); + + SchedulerManager.shutDown(); + + if (result.isSuccess()) { + System.exit(0); + } else { + System.exit(-1); + } + } + + private static void printIndicator(JobResult jobResult) { + if (jobResult.isSuccess()) { + LoggerManager.getPlaintTextLogger().info(CliConstants.SUCCESS_INDICATOR); + } else { + LoggerManager.getPlaintTextLogger().info(jobResult.getMessage()); + StringBuilder b = new StringBuilder(); + for (Map.Entry e : jobResult.getExtraMessage().entrySet()) { + b.append(e.getKey()).append(":").append(e.getValue()).append(System.lineSeparator()); + } + LoggerManager.getPlaintTextLogger().info(b.toString()); + LoggerManager.getPlaintTextLogger().info(CliConstants.FAILURE_INDICATOR); + } + } + + private static Boolean isHelp(CliCtx ctx) { + if (ctx.getVarAccess().hasVar(CliKeys.LINKIS_CLIENT_HELP_OPT)) { + return true; + } + return false; + } + + private static Boolean isVersionCmd(CliCtx ctx) { + if (ctx.getVarAccess().hasVar(CliKeys.VERSION)) { + return true; + } + return false; + } + + private static Boolean isJobCmd(CliCtx ctx) { + if (ctx.getVarAccess().hasVar(CliKeys.LINKIS_CLIENT_KILL_OPT) + || ctx.getVarAccess().hasVar(CliKeys.LINKIS_CLIENT_STATUS_OPT) + || ctx.getVarAccess().hasVar(CliKeys.LINKIS_CLIENT_DESC_OPT) + || ctx.getVarAccess().hasVar(CliKeys.LINKIS_CLIENT_LOG_OPT) + || ctx.getVarAccess().hasVar(CliKeys.LINKIS_CLIENT_RESULT_OPT) + || ctx.getVarAccess().hasVar(CliKeys.LINKIS_CLIENT_LIST_OPT)) { + return true; + } + return false; + } + + private static Boolean isOnceCmd(CliCtx ctx) { + return false; + } +} diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/constants/AppConstants.java b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/constants/CliConstants.java similarity index 61% rename from linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/constants/AppConstants.java rename to linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/constants/CliConstants.java index 711fc2d156..e81196754b 100644 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/constants/AppConstants.java +++ b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/constants/CliConstants.java @@ -17,10 +17,10 @@ package org.apache.linkis.cli.application.constants; -public class AppConstants { +public class CliConstants { public static final String DEFAULT_CONFIG_NAME = "linkis-cli.properties"; - public static final Long JOB_QUERY_SLEEP_MILLS = 2000l; + public static final Long JOB_QUERY_SLEEP_MILLS = 2000L; public static final String RESULTSET_LOGO = "============ RESULT SET {0} ============"; public static final String RESULTSET_META_BEGIN_LOGO = "----------- META DATA ------------"; @@ -41,4 +41,32 @@ public class AppConstants { public static final String UJES_MODE = "ujes"; public static final String ONCE_MODE = "once"; + + public static final Integer REQUEST_MAX_RETRY_TIME = 3; + + public static final String UNIVERSAL_CMD = "linkis-cli"; + + public static final String UNIVERSAL_DESC = "command for all types of jobs supported by Linkis"; + + public static final String VERSION_CMD = "version"; + + public static final String VERSION_DESC = "display version"; + + public static final String SUCCESS_INDICATOR = "############Execute Success!!!########"; + + public static final String FAILURE_INDICATOR = "############Execute Error!!!########"; + + public static final String ARRAY_SEQ = "@#@"; + + public static final String ARRAY_SEQ_REGEX = "(?=([^\"]*\"[^\"]*\")*[^\"]*$)"; + + public static final int MAX_NUM_OF_COMMAND_ARGUEMENTS = 10; + + public static final String CONFIG_DIR = "config.path"; + + public static final String[] CONFIG_EXTENSION = {"properties"}; + + public static final String SYSTEM_PROPERTIES_IDENTIFIER = "SYS_PROP"; + + public static final String SYSTEM_ENV_IDENTIFIER = "SYS_ENV"; } diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/constants/AppKeys.java b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/constants/CliKeys.java similarity index 97% rename from linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/constants/AppKeys.java rename to linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/constants/CliKeys.java index 7a38d9191f..d46cfd6ad8 100644 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/constants/AppKeys.java +++ b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/constants/CliKeys.java @@ -17,7 +17,7 @@ package org.apache.linkis.cli.application.constants; -public class AppKeys { +public class CliKeys { /** User Not configurable */ public static final String ADMIN_USERS = "hadoop,root,shangda"; @@ -38,6 +38,8 @@ public class AppKeys { public static final String DEFAULT_CONFIG_FILE_NAME_KEY = "conf.file"; public static final String LINUX_USER_KEY = "user.name"; + public static final String VERSION = "cli.version"; + /** Configurable */ /* execution type @@ -139,6 +141,6 @@ public class AppKeys { public static final String LINKIS_CLIENT_COMMON_RESULT_SET_PAGE_SIZE = LINKIS_CLIENT_COMMON + ".resultset.page.size"; - public static final String REUSABLE_UJES_CLIENT = "reusable.ujes.client"; - public static final String REUSABLE_ONCEJOB_CLIENT = "reusable.onceJob.client"; + public static final String Linkis_OPER = "linkis.oper"; + public static final String LINKIS_ONCE = "linkis.once"; } diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/constants/LinkisConstants.java b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/constants/LinkisConstants.java similarity index 100% rename from linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/constants/LinkisConstants.java rename to linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/constants/LinkisConstants.java diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/constants/LinkisKeys.java b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/constants/LinkisKeys.java similarity index 97% rename from linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/constants/LinkisKeys.java rename to linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/constants/LinkisKeys.java index 9488c25f4a..bffca826da 100644 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/constants/LinkisKeys.java +++ b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/constants/LinkisKeys.java @@ -46,4 +46,5 @@ public class LinkisKeys { public static final String KEY_YARN_QUEUE = "wds.linkis.rm.yarnqueue"; public static final String KEY_HIVE_RESULT_DISPLAY_TBALE = "hive.resultset.use.unique.column.names"; + public static final String CLI_VERSION = "cli.version"; } diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-common/src/main/java/org/apache/linkis/cli/common/entity/command/CmdOption.java b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/entity/command/CmdOption.java similarity index 95% rename from linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-common/src/main/java/org/apache/linkis/cli/common/entity/command/CmdOption.java rename to linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/entity/command/CmdOption.java index f9b6924be5..1583090c88 100644 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-common/src/main/java/org/apache/linkis/cli/common/entity/command/CmdOption.java +++ b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/entity/command/CmdOption.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.linkis.cli.common.entity.command; +package org.apache.linkis.cli.application.entity.command; public interface CmdOption extends Cloneable { String getParamName(); diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-common/src/main/java/org/apache/linkis/cli/common/entity/command/CmdTemplate.java b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/entity/command/CmdTemplate.java similarity index 94% rename from linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-common/src/main/java/org/apache/linkis/cli/common/entity/command/CmdTemplate.java rename to linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/entity/command/CmdTemplate.java index da802bc5ff..94677774a8 100644 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-common/src/main/java/org/apache/linkis/cli/common/entity/command/CmdTemplate.java +++ b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/entity/command/CmdTemplate.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.linkis.cli.common.entity.command; +package org.apache.linkis.cli.application.entity.command; import java.util.List; import java.util.Map; diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-common/src/main/java/org/apache/linkis/cli/common/entity/command/CmdType.java b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/entity/command/CmdType.java similarity index 93% rename from linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-common/src/main/java/org/apache/linkis/cli/common/entity/command/CmdType.java rename to linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/entity/command/CmdType.java index 6819fdb92b..2f9dd803e8 100644 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-common/src/main/java/org/apache/linkis/cli/common/entity/command/CmdType.java +++ b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/entity/command/CmdType.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.linkis.cli.common.entity.command; +package org.apache.linkis.cli.application.entity.command; public interface CmdType extends Cloneable { diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-common/src/main/java/org/apache/linkis/cli/common/entity/command/ParamItem.java b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/entity/command/ParamItem.java similarity index 96% rename from linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-common/src/main/java/org/apache/linkis/cli/common/entity/command/ParamItem.java rename to linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/entity/command/ParamItem.java index 02a33cfc95..c8a28d9009 100644 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-common/src/main/java/org/apache/linkis/cli/common/entity/command/ParamItem.java +++ b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/entity/command/ParamItem.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.linkis.cli.common.entity.command; +package org.apache.linkis.cli.application.entity.command; public class ParamItem { private String keyPrefix; diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-common/src/main/java/org/apache/linkis/cli/common/entity/command/Params.java b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/entity/command/Params.java similarity index 91% rename from linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-common/src/main/java/org/apache/linkis/cli/common/entity/command/Params.java rename to linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/entity/command/Params.java index 9fa73c01fc..9548db97bc 100644 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-common/src/main/java/org/apache/linkis/cli/common/entity/command/Params.java +++ b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/entity/command/Params.java @@ -15,8 +15,9 @@ * limitations under the License. */ -package org.apache.linkis.cli.common.entity.command; +package org.apache.linkis.cli.application.entity.command; +import java.util.HashMap; import java.util.Map; public class Params { @@ -30,6 +31,11 @@ public class Params { private Map extraProperties; + public Params() { + this.paramItemMap = new HashMap<>(); + this.extraProperties = new HashMap<>(); + } + public Params( String cid, CmdType cmdType, diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/data/PreparedData.java b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/entity/context/CliCtx.java similarity index 62% rename from linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/data/PreparedData.java rename to linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/entity/context/CliCtx.java index bdde39e23e..f18a402c2e 100644 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/data/PreparedData.java +++ b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/entity/context/CliCtx.java @@ -15,20 +15,25 @@ * limitations under the License. */ -package org.apache.linkis.cli.application.data; +package org.apache.linkis.cli.application.entity.context; -import org.apache.linkis.cli.common.entity.properties.ClientProperties; +import org.apache.linkis.cli.application.entity.command.CmdTemplate; +import org.apache.linkis.cli.application.entity.command.CmdType; +import org.apache.linkis.cli.application.entity.var.VarAccess; import java.util.Map; -public class PreparedData { - Map propertiesMap; +public interface CliCtx { + /** + * accessing and passing VarAccess instance between jobs + * + * @return + */ + CmdType getCmdType(); - public PreparedData(Map propertiesMap) { - this.propertiesMap = propertiesMap; - } + CmdTemplate getTemplate(); - public Map getPropertiesMap() { - return propertiesMap; - } + VarAccess getVarAccess(); + + Map getExtraMap(); } diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/present/display/DisplayOperator.java b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/entity/job/Job.java similarity index 79% rename from linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/present/display/DisplayOperator.java rename to linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/entity/job/Job.java index 228b75856d..303c530c2a 100644 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/present/display/DisplayOperator.java +++ b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/entity/job/Job.java @@ -15,10 +15,14 @@ * limitations under the License. */ -package org.apache.linkis.cli.core.present.display; +package org.apache.linkis.cli.application.entity.job; -import org.apache.linkis.cli.core.present.display.data.DisplayData; +import org.apache.linkis.cli.application.entity.context.CliCtx; -public interface DisplayOperator { - void doOutput(DisplayData data); +public interface Job { + void build(CliCtx ctx); + + JobResult run(); + + void onDestroy(); } diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-common/src/main/java/org/apache/linkis/cli/common/entity/job/JobSubType.java b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/entity/job/JobResult.java similarity index 80% rename from linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-common/src/main/java/org/apache/linkis/cli/common/entity/job/JobSubType.java rename to linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/entity/job/JobResult.java index c2c8539e4b..eba7b7ba8a 100644 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-common/src/main/java/org/apache/linkis/cli/common/entity/job/JobSubType.java +++ b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/entity/job/JobResult.java @@ -15,9 +15,14 @@ * limitations under the License. */ -package org.apache.linkis.cli.common.entity.job; +package org.apache.linkis.cli.application.entity.job; -public interface JobSubType { +import java.util.Map; - String getName(); +public interface JobResult { + Boolean isSuccess(); + + String getMessage(); + + Map getExtraMessage(); } diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-common/src/main/java/org/apache/linkis/cli/common/entity/job/JobStatus.java b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/entity/job/JobStatus.java similarity index 95% rename from linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-common/src/main/java/org/apache/linkis/cli/common/entity/job/JobStatus.java rename to linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/entity/job/JobStatus.java index d0b6f445b6..7f048dd88a 100644 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-common/src/main/java/org/apache/linkis/cli/common/entity/job/JobStatus.java +++ b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/entity/job/JobStatus.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.linkis.cli.common.entity.job; +package org.apache.linkis.cli.application.entity.job; public interface JobStatus { diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-common/src/main/java/org/apache/linkis/cli/common/entity/version/Version.java b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/entity/operator/JobOper.java similarity index 90% rename from linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-common/src/main/java/org/apache/linkis/cli/common/entity/version/Version.java rename to linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/entity/operator/JobOper.java index 7f005b0bbd..ae1bf7c465 100644 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-common/src/main/java/org/apache/linkis/cli/common/entity/version/Version.java +++ b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/entity/operator/JobOper.java @@ -15,6 +15,6 @@ * limitations under the License. */ -package org.apache.linkis.cli.common.entity.version; +package org.apache.linkis.cli.application.entity.operator; -public interface Version {} +public interface JobOper {} diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-common/src/main/java/org/apache/linkis/cli/common/entity/present/Model.java b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/entity/present/Model.java similarity index 93% rename from linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-common/src/main/java/org/apache/linkis/cli/common/entity/present/Model.java rename to linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/entity/present/Model.java index b68fd986aa..9fe2c106fc 100644 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-common/src/main/java/org/apache/linkis/cli/common/entity/present/Model.java +++ b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/entity/present/Model.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.linkis.cli.common.entity.present; +package org.apache.linkis.cli.application.entity.present; public interface Model { void buildModel(Object data); diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-common/src/main/java/org/apache/linkis/cli/common/entity/present/Presenter.java b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/entity/present/Presenter.java similarity index 88% rename from linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-common/src/main/java/org/apache/linkis/cli/common/entity/present/Presenter.java rename to linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/entity/present/Presenter.java index dbe1adaefe..7c27316dfb 100644 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-common/src/main/java/org/apache/linkis/cli/common/entity/present/Presenter.java +++ b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/entity/present/Presenter.java @@ -15,8 +15,8 @@ * limitations under the License. */ -package org.apache.linkis.cli.common.entity.present; +package org.apache.linkis.cli.application.entity.present; public interface Presenter { - void present(Model model, PresentWay presentWay); + void present(Model model); } diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-common/src/main/java/org/apache/linkis/cli/common/entity/var/VarAccess.java b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/entity/var/VarAccess.java similarity index 94% rename from linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-common/src/main/java/org/apache/linkis/cli/common/entity/var/VarAccess.java rename to linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/entity/var/VarAccess.java index 50df2236e1..8236bd81c9 100644 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-common/src/main/java/org/apache/linkis/cli/common/entity/var/VarAccess.java +++ b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/entity/var/VarAccess.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.linkis.cli.common.entity.var; +package org.apache.linkis.cli.application.entity.var; /** * @program: linkis-cli @@ -24,8 +24,6 @@ */ public interface VarAccess { - void checkInit(); - T getVar(Class clazz, String key); T getVarOrDefault(Class clazz, String key, T defaultValue); diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/exception/BuilderException.java b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/exception/BuilderException.java similarity index 83% rename from linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/exception/BuilderException.java rename to linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/exception/BuilderException.java index b4f35643aa..b00c15f7e0 100644 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/exception/BuilderException.java +++ b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/exception/BuilderException.java @@ -15,11 +15,10 @@ * limitations under the License. */ -package org.apache.linkis.cli.core.exception; +package org.apache.linkis.cli.application.exception; -import org.apache.linkis.cli.common.exception.LinkisClientRuntimeException; -import org.apache.linkis.cli.common.exception.error.ErrorLevel; -import org.apache.linkis.cli.common.exception.error.ErrorMsg; +import org.apache.linkis.cli.application.exception.error.ErrorLevel; +import org.apache.linkis.cli.application.exception.error.ErrorMsg; public class BuilderException extends LinkisClientRuntimeException { private static final long serialVersionUID = 5454234257L; diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/exception/CommandException.java b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/exception/CommandException.java similarity index 85% rename from linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/exception/CommandException.java rename to linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/exception/CommandException.java index 6f6c5512d4..e0cff74b1e 100644 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/exception/CommandException.java +++ b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/exception/CommandException.java @@ -15,12 +15,11 @@ * limitations under the License. */ -package org.apache.linkis.cli.core.exception; +package org.apache.linkis.cli.application.exception; -import org.apache.linkis.cli.common.entity.command.CmdType; -import org.apache.linkis.cli.common.exception.LinkisClientRuntimeException; -import org.apache.linkis.cli.common.exception.error.ErrorLevel; -import org.apache.linkis.cli.common.exception.error.ErrorMsg; +import org.apache.linkis.cli.application.entity.command.CmdType; +import org.apache.linkis.cli.application.exception.error.ErrorLevel; +import org.apache.linkis.cli.application.exception.error.ErrorMsg; public class CommandException extends LinkisClientRuntimeException { private static final long serialVersionUID = 745261661L; diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-common/src/main/java/org/apache/linkis/cli/common/exception/LinkisClientException.java b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/exception/LinkisClientException.java similarity index 94% rename from linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-common/src/main/java/org/apache/linkis/cli/common/exception/LinkisClientException.java rename to linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/exception/LinkisClientException.java index 7ffd880b73..9dcb69a410 100644 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-common/src/main/java/org/apache/linkis/cli/common/exception/LinkisClientException.java +++ b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/exception/LinkisClientException.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.linkis.cli.common.exception; +package org.apache.linkis.cli.application.exception; public abstract class LinkisClientException extends Exception { private static final long serialVersionUID = 42563456489L; diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/exception/LinkisClientExecutionException.java b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/exception/LinkisClientExecutionException.java similarity index 84% rename from linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/exception/LinkisClientExecutionException.java rename to linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/exception/LinkisClientExecutionException.java index e97d068d4e..3acfd71f99 100644 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/exception/LinkisClientExecutionException.java +++ b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/exception/LinkisClientExecutionException.java @@ -15,11 +15,10 @@ * limitations under the License. */ -package org.apache.linkis.cli.core.exception; +package org.apache.linkis.cli.application.exception; -import org.apache.linkis.cli.common.exception.LinkisClientRuntimeException; -import org.apache.linkis.cli.common.exception.error.ErrorLevel; -import org.apache.linkis.cli.common.exception.error.ErrorMsg; +import org.apache.linkis.cli.application.exception.error.ErrorLevel; +import org.apache.linkis.cli.application.exception.error.ErrorMsg; public class LinkisClientExecutionException extends LinkisClientRuntimeException { private static final long serialVersionUID = 987189405659L; diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-common/src/main/java/org/apache/linkis/cli/common/exception/LinkisClientRuntimeException.java b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/exception/LinkisClientRuntimeException.java similarity index 94% rename from linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-common/src/main/java/org/apache/linkis/cli/common/exception/LinkisClientRuntimeException.java rename to linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/exception/LinkisClientRuntimeException.java index 98538b05e4..0d5f4f7976 100644 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-common/src/main/java/org/apache/linkis/cli/common/exception/LinkisClientRuntimeException.java +++ b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/exception/LinkisClientRuntimeException.java @@ -15,10 +15,10 @@ * limitations under the License. */ -package org.apache.linkis.cli.common.exception; +package org.apache.linkis.cli.application.exception; -import org.apache.linkis.cli.common.exception.error.ErrorLevel; -import org.apache.linkis.cli.common.exception.error.ErrorMsg; +import org.apache.linkis.cli.application.exception.error.ErrorLevel; +import org.apache.linkis.cli.application.exception.error.ErrorMsg; import java.text.MessageFormat; diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/exception/PresenterException.java b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/exception/PresenterException.java similarity index 83% rename from linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/exception/PresenterException.java rename to linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/exception/PresenterException.java index dbfd57a2cc..42aa1b1e17 100644 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/exception/PresenterException.java +++ b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/exception/PresenterException.java @@ -15,11 +15,10 @@ * limitations under the License. */ -package org.apache.linkis.cli.core.exception; +package org.apache.linkis.cli.application.exception; -import org.apache.linkis.cli.common.exception.LinkisClientRuntimeException; -import org.apache.linkis.cli.common.exception.error.ErrorLevel; -import org.apache.linkis.cli.common.exception.error.ErrorMsg; +import org.apache.linkis.cli.application.exception.error.ErrorLevel; +import org.apache.linkis.cli.application.exception.error.ErrorMsg; public class PresenterException extends LinkisClientRuntimeException { private static final long serialVersionUID = 212314213L; diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/exception/PropsException.java b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/exception/PropsException.java similarity index 83% rename from linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/exception/PropsException.java rename to linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/exception/PropsException.java index e7e3799dcc..d39a9de3c0 100644 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/exception/PropsException.java +++ b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/exception/PropsException.java @@ -15,11 +15,10 @@ * limitations under the License. */ -package org.apache.linkis.cli.core.exception; +package org.apache.linkis.cli.application.exception; -import org.apache.linkis.cli.common.exception.LinkisClientRuntimeException; -import org.apache.linkis.cli.common.exception.error.ErrorLevel; -import org.apache.linkis.cli.common.exception.error.ErrorMsg; +import org.apache.linkis.cli.application.exception.error.ErrorLevel; +import org.apache.linkis.cli.application.exception.error.ErrorMsg; public class PropsException extends LinkisClientRuntimeException { private static final long serialVersionUID = 182747823415933L; diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/exception/TransformerException.java b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/exception/TransformerException.java similarity index 83% rename from linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/exception/TransformerException.java rename to linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/exception/TransformerException.java index efb578b971..d5b35c6c8d 100644 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/exception/TransformerException.java +++ b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/exception/TransformerException.java @@ -15,11 +15,10 @@ * limitations under the License. */ -package org.apache.linkis.cli.core.exception; +package org.apache.linkis.cli.application.exception; -import org.apache.linkis.cli.common.exception.LinkisClientRuntimeException; -import org.apache.linkis.cli.common.exception.error.ErrorLevel; -import org.apache.linkis.cli.common.exception.error.ErrorMsg; +import org.apache.linkis.cli.application.exception.error.ErrorLevel; +import org.apache.linkis.cli.application.exception.error.ErrorMsg; public class TransformerException extends LinkisClientRuntimeException { private static final long serialVersionUID = 5454234257L; diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/exception/UnknownException.java b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/exception/UnknownException.java similarity index 83% rename from linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/exception/UnknownException.java rename to linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/exception/UnknownException.java index ff5cfac139..a3fecf3537 100644 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/exception/UnknownException.java +++ b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/exception/UnknownException.java @@ -15,11 +15,10 @@ * limitations under the License. */ -package org.apache.linkis.cli.core.exception; +package org.apache.linkis.cli.application.exception; -import org.apache.linkis.cli.common.exception.LinkisClientRuntimeException; -import org.apache.linkis.cli.common.exception.error.ErrorLevel; -import org.apache.linkis.cli.common.exception.error.ErrorMsg; +import org.apache.linkis.cli.application.exception.error.ErrorLevel; +import org.apache.linkis.cli.application.exception.error.ErrorMsg; public class UnknownException extends LinkisClientRuntimeException { private static final long serialVersionUID = 974159L; diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/exception/ValidateException.java b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/exception/ValidateException.java similarity index 83% rename from linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/exception/ValidateException.java rename to linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/exception/ValidateException.java index 95896d9aa0..f796328dd5 100644 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/exception/ValidateException.java +++ b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/exception/ValidateException.java @@ -15,11 +15,10 @@ * limitations under the License. */ -package org.apache.linkis.cli.core.exception; +package org.apache.linkis.cli.application.exception; -import org.apache.linkis.cli.common.exception.LinkisClientRuntimeException; -import org.apache.linkis.cli.common.exception.error.ErrorLevel; -import org.apache.linkis.cli.common.exception.error.ErrorMsg; +import org.apache.linkis.cli.application.exception.error.ErrorLevel; +import org.apache.linkis.cli.application.exception.error.ErrorMsg; public class ValidateException extends LinkisClientRuntimeException { private static final long serialVersionUID = 5454234257L; diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/exception/VarAccessException.java b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/exception/VarAccessException.java similarity index 83% rename from linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/exception/VarAccessException.java rename to linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/exception/VarAccessException.java index 34509947d4..31dc848e36 100644 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/exception/VarAccessException.java +++ b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/exception/VarAccessException.java @@ -15,11 +15,10 @@ * limitations under the License. */ -package org.apache.linkis.cli.core.exception; +package org.apache.linkis.cli.application.exception; -import org.apache.linkis.cli.common.exception.LinkisClientRuntimeException; -import org.apache.linkis.cli.common.exception.error.ErrorLevel; -import org.apache.linkis.cli.common.exception.error.ErrorMsg; +import org.apache.linkis.cli.application.exception.error.ErrorLevel; +import org.apache.linkis.cli.application.exception.error.ErrorMsg; public class VarAccessException extends LinkisClientRuntimeException { private static final long serialVersionUID = 125344127L; diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/exception/error/CommonErrMsg.java b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/exception/error/CommonErrMsg.java similarity index 96% rename from linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/exception/error/CommonErrMsg.java rename to linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/exception/error/CommonErrMsg.java index 67eb0701c7..ea6247e0c4 100644 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/exception/error/CommonErrMsg.java +++ b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/exception/error/CommonErrMsg.java @@ -15,9 +15,7 @@ * limitations under the License. */ -package org.apache.linkis.cli.core.exception.error; - -import org.apache.linkis.cli.common.exception.error.ErrorMsg; +package org.apache.linkis.cli.application.exception.error; public enum CommonErrMsg implements ErrorMsg { diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-common/src/main/java/org/apache/linkis/cli/common/exception/error/ErrorLevel.java b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/exception/error/ErrorLevel.java similarity index 96% rename from linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-common/src/main/java/org/apache/linkis/cli/common/exception/error/ErrorLevel.java rename to linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/exception/error/ErrorLevel.java index ed2867c72d..170cedf9a5 100644 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-common/src/main/java/org/apache/linkis/cli/common/exception/error/ErrorLevel.java +++ b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/exception/error/ErrorLevel.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.linkis.cli.common.exception.error; +package org.apache.linkis.cli.application.exception.error; public enum ErrorLevel { /** warn 1 error 2 fatal 3 */ diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-common/src/main/java/org/apache/linkis/cli/common/exception/error/ErrorMsg.java b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/exception/error/ErrorMsg.java similarity index 93% rename from linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-common/src/main/java/org/apache/linkis/cli/common/exception/error/ErrorMsg.java rename to linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/exception/error/ErrorMsg.java index 3d092fe67d..4b6c5d3244 100644 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-common/src/main/java/org/apache/linkis/cli/common/exception/error/ErrorMsg.java +++ b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/exception/error/ErrorMsg.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.linkis.cli.common.exception.error; +package org.apache.linkis.cli.application.exception.error; public interface ErrorMsg { diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/interactor/command/LinkisCmdType.java b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/command/CliCmdType.java similarity index 78% rename from linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/interactor/command/LinkisCmdType.java rename to linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/command/CliCmdType.java index f03c3e779b..ca8986fefc 100644 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/interactor/command/LinkisCmdType.java +++ b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/command/CliCmdType.java @@ -17,23 +17,23 @@ package org.apache.linkis.cli.application.interactor.command; -import org.apache.linkis.cli.common.entity.command.CmdType; -import org.apache.linkis.cli.core.constants.CommonConstants; +import org.apache.linkis.cli.application.constants.CliConstants; +import org.apache.linkis.cli.application.entity.command.CmdType; -public enum LinkisCmdType implements CmdType { - UNIVERSAL(CommonConstants.UNIVERSAL_SUBCMD, 1, CommonConstants.UNIVERSAL_SUBCMD_DESC); +public enum CliCmdType implements CmdType { + UNIVERSAL(CliConstants.UNIVERSAL_CMD, 1, CliConstants.UNIVERSAL_DESC); private int id; private String name; private String desc; - LinkisCmdType(String name, int id) { + CliCmdType(String name, int id) { this.id = id; this.name = name; this.desc = null; } - LinkisCmdType(String name, int id, String desc) { + CliCmdType(String name, int id, String desc) { this.id = id; this.name = name; this.desc = desc; diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/command/CmdTemplateFactory.java b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/command/CmdTemplateFactory.java similarity index 83% rename from linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/command/CmdTemplateFactory.java rename to linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/command/CmdTemplateFactory.java index 91a9f796cb..f3b901c1f8 100644 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/command/CmdTemplateFactory.java +++ b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/command/CmdTemplateFactory.java @@ -15,13 +15,13 @@ * limitations under the License. */ -package org.apache.linkis.cli.core.interactor.command; +package org.apache.linkis.cli.application.interactor.command; -import org.apache.linkis.cli.common.entity.command.CmdTemplate; -import org.apache.linkis.cli.common.entity.command.CmdType; -import org.apache.linkis.cli.common.exception.error.ErrorLevel; -import org.apache.linkis.cli.core.exception.CommandException; -import org.apache.linkis.cli.core.exception.error.CommonErrMsg; +import org.apache.linkis.cli.application.entity.command.CmdTemplate; +import org.apache.linkis.cli.application.entity.command.CmdType; +import org.apache.linkis.cli.application.exception.CommandException; +import org.apache.linkis.cli.application.exception.error.CommonErrMsg; +import org.apache.linkis.cli.application.exception.error.ErrorLevel; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/command/SpecialMap.java b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/command/SpecialMap.java similarity index 94% rename from linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/command/SpecialMap.java rename to linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/command/SpecialMap.java index 86075e3328..427c35baef 100644 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/command/SpecialMap.java +++ b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/command/SpecialMap.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.linkis.cli.core.interactor.command; +package org.apache.linkis.cli.application.interactor.command; import java.util.HashMap; import java.util.Map; diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/command/fitter/AbstractFitter.java b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/command/fitter/AbstractFitter.java similarity index 89% rename from linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/command/fitter/AbstractFitter.java rename to linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/command/fitter/AbstractFitter.java index d32d05bbb6..f5e1a99b58 100644 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/command/fitter/AbstractFitter.java +++ b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/command/fitter/AbstractFitter.java @@ -15,18 +15,18 @@ * limitations under the License. */ -package org.apache.linkis.cli.core.interactor.command.fitter; - -import org.apache.linkis.cli.common.entity.command.CmdOption; -import org.apache.linkis.cli.common.entity.command.CmdTemplate; -import org.apache.linkis.cli.common.entity.command.CmdType; -import org.apache.linkis.cli.common.exception.LinkisClientRuntimeException; -import org.apache.linkis.cli.common.exception.error.ErrorLevel; -import org.apache.linkis.cli.core.constants.CommonConstants; -import org.apache.linkis.cli.core.exception.CommandException; -import org.apache.linkis.cli.core.exception.error.CommonErrMsg; -import org.apache.linkis.cli.core.interactor.command.template.option.Flag; -import org.apache.linkis.cli.core.interactor.command.template.option.Parameter; +package org.apache.linkis.cli.application.interactor.command.fitter; + +import org.apache.linkis.cli.application.constants.CliConstants; +import org.apache.linkis.cli.application.entity.command.CmdOption; +import org.apache.linkis.cli.application.entity.command.CmdTemplate; +import org.apache.linkis.cli.application.entity.command.CmdType; +import org.apache.linkis.cli.application.exception.CommandException; +import org.apache.linkis.cli.application.exception.LinkisClientRuntimeException; +import org.apache.linkis.cli.application.exception.error.CommonErrMsg; +import org.apache.linkis.cli.application.exception.error.ErrorLevel; +import org.apache.linkis.cli.application.interactor.command.template.option.Flag; +import org.apache.linkis.cli.application.interactor.command.template.option.Parameter; import org.apache.commons.lang3.StringUtils; @@ -225,7 +225,7 @@ private final int setParameterValue( Parameter param = (Parameter) cmdOption; if (param.accepctArrayValue()) { String[] args2 = Arrays.copyOfRange(args, argIdx, args.length); - param.setValueWithStr(StringUtils.join(args2, CommonConstants.ARRAY_SEQ)); + param.setValueWithStr(StringUtils.join(args2, CliConstants.ARRAY_SEQ)); return args.length; } else { parameters.get(paraIdx).setValueWithStr(args[argIdx]); diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/command/fitter/Fitter.java b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/command/fitter/Fitter.java similarity index 80% rename from linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/command/fitter/Fitter.java rename to linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/command/fitter/Fitter.java index 305ffcaef1..cfd0f0fcfa 100644 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/command/fitter/Fitter.java +++ b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/command/fitter/Fitter.java @@ -15,10 +15,10 @@ * limitations under the License. */ -package org.apache.linkis.cli.core.interactor.command.fitter; +package org.apache.linkis.cli.application.interactor.command.fitter; -import org.apache.linkis.cli.common.entity.command.CmdTemplate; -import org.apache.linkis.cli.common.exception.LinkisClientRuntimeException; +import org.apache.linkis.cli.application.entity.command.CmdTemplate; +import org.apache.linkis.cli.application.exception.LinkisClientRuntimeException; public interface Fitter { FitterResult fit(String[] input, CmdTemplate templateCopy) throws LinkisClientRuntimeException; diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/command/fitter/FitterResult.java b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/command/fitter/FitterResult.java similarity index 90% rename from linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/command/fitter/FitterResult.java rename to linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/command/fitter/FitterResult.java index 5e9f9820b1..abcee0d9af 100644 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/command/fitter/FitterResult.java +++ b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/command/fitter/FitterResult.java @@ -15,9 +15,9 @@ * limitations under the License. */ -package org.apache.linkis.cli.core.interactor.command.fitter; +package org.apache.linkis.cli.application.interactor.command.fitter; -import org.apache.linkis.cli.common.entity.command.CmdTemplate; +import org.apache.linkis.cli.application.entity.command.CmdTemplate; public class FitterResult { String[] remains; diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/command/fitter/FitterUtils.java b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/command/fitter/FitterUtils.java similarity index 93% rename from linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/command/fitter/FitterUtils.java rename to linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/command/fitter/FitterUtils.java index 40d80d84b2..1b6b46cd10 100644 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/command/fitter/FitterUtils.java +++ b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/command/fitter/FitterUtils.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.linkis.cli.core.interactor.command.fitter; +package org.apache.linkis.cli.application.interactor.command.fitter; public class FitterUtils { diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/command/fitter/SingleTplFitter.java b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/command/fitter/SingleTplFitter.java similarity index 76% rename from linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/command/fitter/SingleTplFitter.java rename to linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/command/fitter/SingleTplFitter.java index 8eed2ea89f..5b9f6eeb1a 100644 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/command/fitter/SingleTplFitter.java +++ b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/command/fitter/SingleTplFitter.java @@ -15,13 +15,13 @@ * limitations under the License. */ -package org.apache.linkis.cli.core.interactor.command.fitter; +package org.apache.linkis.cli.application.interactor.command.fitter; -import org.apache.linkis.cli.common.entity.command.CmdTemplate; -import org.apache.linkis.cli.common.exception.LinkisClientRuntimeException; -import org.apache.linkis.cli.common.exception.error.ErrorLevel; -import org.apache.linkis.cli.core.exception.CommandException; -import org.apache.linkis.cli.core.exception.error.CommonErrMsg; +import org.apache.linkis.cli.application.entity.command.CmdTemplate; +import org.apache.linkis.cli.application.exception.CommandException; +import org.apache.linkis.cli.application.exception.LinkisClientRuntimeException; +import org.apache.linkis.cli.application.exception.error.CommonErrMsg; +import org.apache.linkis.cli.application.exception.error.ErrorLevel; import java.util.ArrayList; import java.util.List; @@ -36,7 +36,7 @@ public class SingleTplFitter extends AbstractFitter { public FitterResult fit(String[] input, CmdTemplate templateCopy) throws LinkisClientRuntimeException { - if (input == null || templateCopy == null || input.length == 0) { + if (input == null || templateCopy == null) { throw new CommandException( "CMD0009", ErrorLevel.ERROR, CommonErrMsg.TemplateFitErr, "input or template is null"); } diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/command/parser/AbstarctParser.java b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/command/parser/AbstarctParser.java similarity index 85% rename from linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/command/parser/AbstarctParser.java rename to linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/command/parser/AbstarctParser.java index 30c0e1f014..16107b1869 100644 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/command/parser/AbstarctParser.java +++ b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/command/parser/AbstarctParser.java @@ -15,20 +15,20 @@ * limitations under the License. */ -package org.apache.linkis.cli.core.interactor.command.parser; - -import org.apache.linkis.cli.common.entity.command.CmdOption; -import org.apache.linkis.cli.common.entity.command.CmdTemplate; -import org.apache.linkis.cli.common.entity.command.ParamItem; -import org.apache.linkis.cli.common.entity.command.Params; -import org.apache.linkis.cli.common.exception.error.ErrorLevel; -import org.apache.linkis.cli.core.exception.CommandException; -import org.apache.linkis.cli.core.exception.TransformerException; -import org.apache.linkis.cli.core.exception.error.CommonErrMsg; -import org.apache.linkis.cli.core.interactor.command.SpecialMap; -import org.apache.linkis.cli.core.interactor.command.fitter.Fitter; -import org.apache.linkis.cli.core.interactor.command.parser.result.ParseResult; -import org.apache.linkis.cli.core.interactor.command.parser.transformer.ParamKeyMapper; +package org.apache.linkis.cli.application.interactor.command.parser; + +import org.apache.linkis.cli.application.entity.command.CmdOption; +import org.apache.linkis.cli.application.entity.command.CmdTemplate; +import org.apache.linkis.cli.application.entity.command.ParamItem; +import org.apache.linkis.cli.application.entity.command.Params; +import org.apache.linkis.cli.application.exception.CommandException; +import org.apache.linkis.cli.application.exception.TransformerException; +import org.apache.linkis.cli.application.exception.error.CommonErrMsg; +import org.apache.linkis.cli.application.exception.error.ErrorLevel; +import org.apache.linkis.cli.application.interactor.command.SpecialMap; +import org.apache.linkis.cli.application.interactor.command.fitter.Fitter; +import org.apache.linkis.cli.application.interactor.command.parser.result.ParseResult; +import org.apache.linkis.cli.application.interactor.command.parser.transformer.ParamKeyMapper; import org.apache.commons.lang3.StringUtils; diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/command/parser/Parser.java b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/command/parser/Parser.java similarity index 83% rename from linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/command/parser/Parser.java rename to linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/command/parser/Parser.java index 0f3c847740..0e10977a63 100644 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/command/parser/Parser.java +++ b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/command/parser/Parser.java @@ -15,10 +15,10 @@ * limitations under the License. */ -package org.apache.linkis.cli.core.interactor.command.parser; +package org.apache.linkis.cli.application.interactor.command.parser; -import org.apache.linkis.cli.common.entity.command.Params; -import org.apache.linkis.cli.core.interactor.command.parser.result.ParseResult; +import org.apache.linkis.cli.application.entity.command.Params; +import org.apache.linkis.cli.application.interactor.command.parser.result.ParseResult; /** * 1. parse cmd arguments and fill into template 2. generate unique identifier for sub command 3. diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/command/parser/SingleCmdParser.java b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/command/parser/SingleCmdParser.java similarity index 69% rename from linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/command/parser/SingleCmdParser.java rename to linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/command/parser/SingleCmdParser.java index aea3086fd5..663d842191 100644 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/command/parser/SingleCmdParser.java +++ b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/command/parser/SingleCmdParser.java @@ -15,15 +15,15 @@ * limitations under the License. */ -package org.apache.linkis.cli.core.interactor.command.parser; +package org.apache.linkis.cli.application.interactor.command.parser; -import org.apache.linkis.cli.common.entity.command.CmdTemplate; -import org.apache.linkis.cli.common.entity.command.Params; -import org.apache.linkis.cli.common.exception.error.ErrorLevel; -import org.apache.linkis.cli.core.exception.CommandException; -import org.apache.linkis.cli.core.exception.error.CommonErrMsg; -import org.apache.linkis.cli.core.interactor.command.fitter.FitterResult; -import org.apache.linkis.cli.core.interactor.command.parser.result.ParseResult; +import org.apache.linkis.cli.application.entity.command.CmdTemplate; +import org.apache.linkis.cli.application.entity.command.Params; +import org.apache.linkis.cli.application.exception.CommandException; +import org.apache.linkis.cli.application.exception.error.CommonErrMsg; +import org.apache.linkis.cli.application.exception.error.ErrorLevel; +import org.apache.linkis.cli.application.interactor.command.fitter.FitterResult; +import org.apache.linkis.cli.application.interactor.command.parser.result.ParseResult; import java.util.Arrays; @@ -37,14 +37,10 @@ public class SingleCmdParser extends AbstarctParser { public ParseResult parse(String[] input) { checkInit(); - if (input == null || input.length == 0) { - throw new CommandException( - "CMD0015", - ErrorLevel.ERROR, - CommonErrMsg.ParserParseErr, - template.getCmdType(), - "nothing to parse"); - } + // if (input == null || input.length == 0) { + // logger.info("Nothing to parse"); + // return new ParseResult(template, new Params(), new String[0]); + // } FitterResult result = fitter.fit(input, template); diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/command/parser/result/ParseResult.java b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/command/parser/result/ParseResult.java similarity index 88% rename from linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/command/parser/result/ParseResult.java rename to linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/command/parser/result/ParseResult.java index 2a528f3b86..75059c86c5 100644 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/command/parser/result/ParseResult.java +++ b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/command/parser/result/ParseResult.java @@ -15,10 +15,10 @@ * limitations under the License. */ -package org.apache.linkis.cli.core.interactor.command.parser.result; +package org.apache.linkis.cli.application.interactor.command.parser.result; -import org.apache.linkis.cli.common.entity.command.CmdTemplate; -import org.apache.linkis.cli.common.entity.command.Params; +import org.apache.linkis.cli.application.entity.command.CmdTemplate; +import org.apache.linkis.cli.application.entity.command.Params; public class ParseResult { CmdTemplate parsedTemplate; diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/command/parser/transformer/ParamKeyMapper.java b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/command/parser/transformer/ParamKeyMapper.java similarity index 88% rename from linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/command/parser/transformer/ParamKeyMapper.java rename to linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/command/parser/transformer/ParamKeyMapper.java index 3799906bd5..f162a1c84c 100644 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/command/parser/transformer/ParamKeyMapper.java +++ b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/command/parser/transformer/ParamKeyMapper.java @@ -15,13 +15,13 @@ * limitations under the License. */ -package org.apache.linkis.cli.core.interactor.command.parser.transformer; +package org.apache.linkis.cli.application.interactor.command.parser.transformer; -import org.apache.linkis.cli.common.exception.LinkisClientRuntimeException; -import org.apache.linkis.cli.common.exception.error.ErrorLevel; -import org.apache.linkis.cli.core.exception.CommandException; -import org.apache.linkis.cli.core.exception.error.CommonErrMsg; -import org.apache.linkis.cli.core.utils.CommonUtils; +import org.apache.linkis.cli.application.exception.CommandException; +import org.apache.linkis.cli.application.exception.LinkisClientRuntimeException; +import org.apache.linkis.cli.application.exception.error.CommonErrMsg; +import org.apache.linkis.cli.application.exception.error.ErrorLevel; +import org.apache.linkis.cli.application.utils.CliUtils; import org.apache.commons.lang3.StringUtils; @@ -79,7 +79,7 @@ public void updateMapping(String key, String targetKey) { /** update keyMapping according to kv-String. */ private void updateMappingbyConfig(String kvString) { if (StringUtils.isNotBlank(kvString)) { - Map result = CommonUtils.parseKVStringToMap(kvString, ","); + Map result = CliUtils.parseKVStringToMap(kvString, ","); this.mapperRules.putAll(result); } } diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/command/template/AbstractCmdTemplate.java b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/command/template/AbstractCmdTemplate.java similarity index 91% rename from linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/command/template/AbstractCmdTemplate.java rename to linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/command/template/AbstractCmdTemplate.java index 992b58effc..ec1f318c2b 100644 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/command/template/AbstractCmdTemplate.java +++ b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/command/template/AbstractCmdTemplate.java @@ -15,20 +15,20 @@ * limitations under the License. */ -package org.apache.linkis.cli.core.interactor.command.template; - -import org.apache.linkis.cli.common.entity.command.CmdOption; -import org.apache.linkis.cli.common.entity.command.CmdTemplate; -import org.apache.linkis.cli.common.entity.command.CmdType; -import org.apache.linkis.cli.common.exception.LinkisClientRuntimeException; -import org.apache.linkis.cli.common.exception.error.ErrorLevel; -import org.apache.linkis.cli.core.constants.CommonConstants; -import org.apache.linkis.cli.core.exception.CommandException; -import org.apache.linkis.cli.core.exception.error.CommonErrMsg; -import org.apache.linkis.cli.core.interactor.command.SpecialMap; -import org.apache.linkis.cli.core.interactor.command.template.option.*; -import org.apache.linkis.cli.core.utils.converter.AbstractStringConverter; -import org.apache.linkis.cli.core.utils.converter.PredefinedStringConverters; +package org.apache.linkis.cli.application.interactor.command.template; + +import org.apache.linkis.cli.application.constants.CliConstants; +import org.apache.linkis.cli.application.entity.command.CmdOption; +import org.apache.linkis.cli.application.entity.command.CmdTemplate; +import org.apache.linkis.cli.application.entity.command.CmdType; +import org.apache.linkis.cli.application.exception.CommandException; +import org.apache.linkis.cli.application.exception.LinkisClientRuntimeException; +import org.apache.linkis.cli.application.exception.error.CommonErrMsg; +import org.apache.linkis.cli.application.exception.error.ErrorLevel; +import org.apache.linkis.cli.application.interactor.command.SpecialMap; +import org.apache.linkis.cli.application.interactor.command.template.converter.AbstractStringConverter; +import org.apache.linkis.cli.application.interactor.command.template.converter.PredefinedStringConverters; +import org.apache.linkis.cli.application.interactor.command.template.option.*; import java.lang.reflect.Field; import java.util.*; @@ -291,10 +291,10 @@ protected final SpecialMapOption speciaMapOption( private void checkIllegalOption(final String[] names) { if (names == null || names.length <= 0) { throw new IllegalArgumentException("At least one cmdType should be given to CmdOption."); - } else if (names.length > CommonConstants.MAX_NUM_OF_COMMAND_ARGUEMENTS) { + } else if (names.length > CliConstants.MAX_NUM_OF_COMMAND_ARGUEMENTS) { throw new IllegalArgumentException( "At most " - + CommonConstants.MAX_NUM_OF_COMMAND_ARGUEMENTS + + CliConstants.MAX_NUM_OF_COMMAND_ARGUEMENTS + " cmdType can be given to CmdOption."); } else { for (String name : names) { diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/interactor/command/template/UniversalCmdTemplate.java b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/command/template/UniversalCmdTemplate.java similarity index 75% rename from linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/interactor/command/template/UniversalCmdTemplate.java rename to linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/command/template/UniversalCmdTemplate.java index e238038aea..704449a53c 100644 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/interactor/command/template/UniversalCmdTemplate.java +++ b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/command/template/UniversalCmdTemplate.java @@ -17,15 +17,14 @@ package org.apache.linkis.cli.application.interactor.command.template; -import org.apache.linkis.cli.application.constants.AppConstants; -import org.apache.linkis.cli.application.constants.AppKeys; -import org.apache.linkis.cli.application.interactor.command.LinkisCmdType; -import org.apache.linkis.cli.common.exception.error.ErrorLevel; -import org.apache.linkis.cli.core.exception.CommandException; -import org.apache.linkis.cli.core.exception.ValidateException; -import org.apache.linkis.cli.core.exception.error.CommonErrMsg; -import org.apache.linkis.cli.core.interactor.command.template.AbstractCmdTemplate; -import org.apache.linkis.cli.core.interactor.command.template.option.*; +import org.apache.linkis.cli.application.constants.CliConstants; +import org.apache.linkis.cli.application.constants.CliKeys; +import org.apache.linkis.cli.application.exception.CommandException; +import org.apache.linkis.cli.application.exception.ValidateException; +import org.apache.linkis.cli.application.exception.error.CommonErrMsg; +import org.apache.linkis.cli.application.exception.error.ErrorLevel; +import org.apache.linkis.cli.application.interactor.command.CliCmdType; +import org.apache.linkis.cli.application.interactor.command.template.option.*; import org.apache.commons.lang3.StringUtils; @@ -36,100 +35,100 @@ public class UniversalCmdTemplate extends AbstractCmdTemplate implements Cloneab protected StdOption gatewayUrl = option( - AppKeys.LINKIS_CLIENT_COMMON, - AppKeys.LINKIS_COMMON_GATEWAY_URL, + CliKeys.LINKIS_CLIENT_COMMON, + CliKeys.LINKIS_COMMON_GATEWAY_URL, new String[] {"--gatewayUrl"}, "specify linkis gateway url", true, ""); protected StdOption authenticatationStrategy = option( - AppKeys.LINKIS_CLIENT_COMMON, - AppKeys.LINKIS_COMMON_AUTHENTICATION_STRATEGY, + CliKeys.LINKIS_CLIENT_COMMON, + CliKeys.LINKIS_COMMON_AUTHENTICATION_STRATEGY, new String[] {"--authStg"}, "specify linkis authentication strategy", true, ""); protected StdOption authKey = option( - AppKeys.LINKIS_CLIENT_COMMON, - AppKeys.LINKIS_COMMON_TOKEN_KEY, + CliKeys.LINKIS_CLIENT_COMMON, + CliKeys.LINKIS_COMMON_TOKEN_KEY, new String[] {"--authKey"}, "specify linkis authentication key(tokenKey)", true, ""); protected StdOption authValue = option( - AppKeys.LINKIS_CLIENT_COMMON, - AppKeys.LINKIS_COMMON_TOKEN_VALUE, + CliKeys.LINKIS_CLIENT_COMMON, + CliKeys.LINKIS_COMMON_TOKEN_VALUE, new String[] {"--authVal"}, "specify linkis authentication value(tokenValue)", true, ""); protected StdOption userConfigPath = option( - AppKeys.LINKIS_CLIENT_COMMON, - AppKeys.LINKIS_CLIENT_USER_CONFIG, + CliKeys.LINKIS_CLIENT_COMMON, + CliKeys.LINKIS_CLIENT_USER_CONFIG, new String[] {"--userConf"}, "specify user configuration file path(absolute)", true, ""); protected StdOption killOpt = option( - AppKeys.LINKIS_CLIENT_COMMON, - AppKeys.LINKIS_CLIENT_KILL_OPT, + CliKeys.LINKIS_CLIENT_COMMON, + CliKeys.LINKIS_CLIENT_KILL_OPT, new String[] {"--kill"}, "specify linkis taskId for job to be killed", true, ""); protected StdOption logOpt = option( - AppKeys.LINKIS_CLIENT_COMMON, - AppKeys.LINKIS_CLIENT_LOG_OPT, + CliKeys.LINKIS_CLIENT_COMMON, + CliKeys.LINKIS_CLIENT_LOG_OPT, new String[] {"--log"}, "specify linkis taskId for querying job status", true, ""); protected StdOption resultOpt = option( - AppKeys.LINKIS_CLIENT_COMMON, - AppKeys.LINKIS_CLIENT_RESULT_OPT, + CliKeys.LINKIS_CLIENT_COMMON, + CliKeys.LINKIS_CLIENT_RESULT_OPT, new String[] {"--result"}, "specify linkis taskId for querying job status", true, ""); protected StdOption statusOpt = option( - AppKeys.LINKIS_CLIENT_COMMON, - AppKeys.LINKIS_CLIENT_STATUS_OPT, + CliKeys.LINKIS_CLIENT_COMMON, + CliKeys.LINKIS_CLIENT_STATUS_OPT, new String[] {"--status"}, "specify linkis taskId for querying job status", true, ""); protected StdOption asyncOpt = option( - AppKeys.LINKIS_CLIENT_COMMON, - AppKeys.LINKIS_CLIENT_ASYNC_OPT, + CliKeys.LINKIS_CLIENT_COMMON, + CliKeys.LINKIS_CLIENT_ASYNC_OPT, new String[] {"--async"}, "specify linkis taskId for querying job status", true, false); protected StdOption modeOpt = option( - AppKeys.LINKIS_CLIENT_COMMON, - AppKeys.LINKIS_CLIENT_MODE_OPT, + CliKeys.LINKIS_CLIENT_COMMON, + CliKeys.LINKIS_CLIENT_MODE_OPT, new String[] {"--mode"}, "specify linkis execution mode: " - + AppConstants.UJES_MODE + + CliConstants.UJES_MODE + "/" - + AppConstants.ONCE_MODE + + CliConstants.ONCE_MODE + ".", true, - AppConstants.UJES_MODE); + CliConstants.UJES_MODE); protected Flag helpOpt = flag( - AppKeys.LINKIS_CLIENT_COMMON, - AppKeys.LINKIS_CLIENT_HELP_OPT, + CliKeys.LINKIS_CLIENT_COMMON, + CliKeys.LINKIS_CLIENT_HELP_OPT, new String[] {"--help"}, "specify linkis taskId for querying job status", true, @@ -137,8 +136,8 @@ public class UniversalCmdTemplate extends AbstractCmdTemplate implements Cloneab protected StdOption engineTypeOP = option( - AppKeys.JOB_LABEL, - AppKeys.JOB_LABEL_ENGINE_TYPE, + CliKeys.JOB_LABEL, + CliKeys.JOB_LABEL_ENGINE_TYPE, new String[] {"-engineType"}, "specify linkis engineType for this job", true, @@ -146,16 +145,16 @@ public class UniversalCmdTemplate extends AbstractCmdTemplate implements Cloneab protected StdOption codeTypeOp = option( - AppKeys.JOB_LABEL, - AppKeys.JOB_LABEL_CODE_TYPE, + CliKeys.JOB_LABEL, + CliKeys.JOB_LABEL_CODE_TYPE, new String[] {"-codeType"}, "specify linkis runType for this job", true, ""); protected StdOption codePathOp = option( - AppKeys.LINKIS_CLIENT_COMMON, - AppKeys.JOB_COMMON_CODE_PATH, + CliKeys.LINKIS_CLIENT_COMMON, + CliKeys.JOB_COMMON_CODE_PATH, new String[] {"-codePath"}, "specify file path that contains code you want to execute", true, @@ -163,8 +162,8 @@ public class UniversalCmdTemplate extends AbstractCmdTemplate implements Cloneab protected StdOption codeOp = option( - AppKeys.JOB_EXEC, - AppKeys.JOB_EXEC_CODE, + CliKeys.JOB_EXEC, + CliKeys.JOB_EXEC_CODE, new String[] {"-code"}, "specify code that you want to execute", true, @@ -172,8 +171,8 @@ public class UniversalCmdTemplate extends AbstractCmdTemplate implements Cloneab protected StdOption scriptPathOp = option( - AppKeys.JOB_SOURCE, - AppKeys.JOB_SOURCE_SCRIPT_PATH, + CliKeys.JOB_SOURCE, + CliKeys.JOB_SOURCE_SCRIPT_PATH, new String[] {"-scriptPath"}, "specify remote path for your uploaded script", true, @@ -181,8 +180,8 @@ public class UniversalCmdTemplate extends AbstractCmdTemplate implements Cloneab protected StdOption submitUser = option( - AppKeys.LINKIS_CLIENT_COMMON, - AppKeys.JOB_COMMON_SUBMIT_USER, + CliKeys.LINKIS_CLIENT_COMMON, + CliKeys.JOB_COMMON_SUBMIT_USER, new String[] {"-submitUser"}, "specify submit user for this job", true, @@ -190,8 +189,8 @@ public class UniversalCmdTemplate extends AbstractCmdTemplate implements Cloneab protected StdOption proxyUser = option( - AppKeys.LINKIS_CLIENT_COMMON, - AppKeys.JOB_COMMON_PROXY_USER, + CliKeys.LINKIS_CLIENT_COMMON, + CliKeys.JOB_COMMON_PROXY_USER, new String[] {"-proxyUser"}, "specify proxy user who executes your code in Linkis server-side", true, @@ -199,8 +198,8 @@ public class UniversalCmdTemplate extends AbstractCmdTemplate implements Cloneab protected StdOption creatorOp = option( - AppKeys.LINKIS_CLIENT_COMMON, - AppKeys.JOB_COMMON_CREATOR, + CliKeys.LINKIS_CLIENT_COMMON, + CliKeys.JOB_COMMON_CREATOR, new String[] {"-creator"}, "specify creator for this job", true, @@ -208,8 +207,8 @@ public class UniversalCmdTemplate extends AbstractCmdTemplate implements Cloneab protected StdOption outPathOp = option( - AppKeys.LINKIS_CLIENT_COMMON, - AppKeys.LINKIS_CLIENT_COMMON_OUTPUT_PATH, + CliKeys.LINKIS_CLIENT_COMMON, + CliKeys.LINKIS_CLIENT_COMMON_OUTPUT_PATH, new String[] {"-outPath"}, "specify output path for resultSet. If not specified, then output reset to screen(stdout)", true, @@ -217,67 +216,79 @@ public class UniversalCmdTemplate extends AbstractCmdTemplate implements Cloneab protected MapOption confMapOp = mapOption( - AppKeys.JOB_PARAM_CONF, - AppKeys.JOB_PARAM_CONF, + CliKeys.JOB_PARAM_CONF, + CliKeys.JOB_PARAM_CONF, new String[] {"-confMap"}, "specify configurationMap(startupMap) for your job. You can put any start-up parameters into this Map(e.g. spark.executor.instances). Input format: -confMap key1=value1 -confMap key2=value2", true); protected MapOption runtimeMapOp = mapOption( - AppKeys.JOB_PARAM_RUNTIME, - AppKeys.JOB_PARAM_RUNTIME, + CliKeys.JOB_PARAM_RUNTIME, + CliKeys.JOB_PARAM_RUNTIME, new String[] {"-runtimeMap"}, "specify runtimeMap for your job. You can put any start-up parameters into this Map(e.g. spark.executor.instances). Input format: -runtimeMap key1=value1 -runtimeMap key2=value2", true); protected SpecialMapOption varMapOp = speciaMapOption( - AppKeys.JOB_PARAM_VAR, - AppKeys.JOB_PARAM_VAR, + CliKeys.JOB_PARAM_VAR, + CliKeys.JOB_PARAM_VAR, new String[] {"-varMap"}, "specify variables map. Variables is for key-word substitution. Use \'${key}\' to specify key-word. Input substitution rule as follow: -varMap key1=value1 -varMap key2=value2", true); protected MapOption labelMapOp = mapOption( - AppKeys.JOB_LABEL, - AppKeys.JOB_LABEL, + CliKeys.JOB_LABEL, + CliKeys.JOB_LABEL, new String[] {"-labelMap"}, "specify label map. You can put any Linkis into this Map. Input format: -labelMap labelName1=labelValue1 -labelMap labelName2=labelValue2", true); protected MapOption sourceMapOp = mapOption( - AppKeys.JOB_SOURCE, - AppKeys.JOB_SOURCE, + CliKeys.JOB_SOURCE, + CliKeys.JOB_SOURCE, new String[] {"-sourceMap"}, "specify source map. Input format: -sourceMap key1=value1 -sourceMap key2=value2", true); protected MapOption jobContentMapOp = mapOption( - AppKeys.JOB_CONTENT, - AppKeys.JOB_CONTENT, + CliKeys.JOB_CONTENT, + CliKeys.JOB_CONTENT, new String[] {"-jobContentMap"}, "specify jobContent map. Input format: -jobContentMap key1=value1 -jobContentMap key2=value2", true); + protected Flag versionFlag = + flag( + CliKeys.VERSION, + CliKeys.VERSION, + new String[] {"--version"}, + "show version", + true, + false); + protected Parameter argumentsParas = parameter( - AppKeys.LINKIS_CLIENT_COMMON, - AppKeys.JOB_EXTRA_ARGUMENTS, + CliKeys.LINKIS_CLIENT_COMMON, + CliKeys.JOB_EXTRA_ARGUMENTS, "arguments", "specify arguments if exist any", true, new String[] {""}); public UniversalCmdTemplate() { - super(LinkisCmdType.UNIVERSAL); + super(CliCmdType.UNIVERSAL); } @Override public void checkParams() throws CommandException { + if (versionFlag.hasVal()) { + return; + } int cnt = 0; if (statusOpt.hasVal()) { cnt++; @@ -349,7 +360,7 @@ public void checkParams() throws CommandException { cnt2++; } if (!modeOpt.hasVal() - || StringUtils.equalsIgnoreCase(modeOpt.getValue(), AppConstants.UJES_MODE)) { + || StringUtils.equalsIgnoreCase(modeOpt.getValue(), CliConstants.UJES_MODE)) { if (cnt2 > 1) { throw new ValidateException( "VLD0001", diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/utils/converter/AbstractStringConverter.java b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/command/template/converter/AbstractStringConverter.java similarity index 91% rename from linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/utils/converter/AbstractStringConverter.java rename to linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/command/template/converter/AbstractStringConverter.java index 3d5afaa172..68f3fae396 100644 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/utils/converter/AbstractStringConverter.java +++ b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/command/template/converter/AbstractStringConverter.java @@ -15,6 +15,6 @@ * limitations under the License. */ -package org.apache.linkis.cli.core.utils.converter; +package org.apache.linkis.cli.application.interactor.command.template.converter; public abstract class AbstractStringConverter implements Converter {} diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/utils/converter/Converter.java b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/command/template/converter/Converter.java similarity index 91% rename from linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/utils/converter/Converter.java rename to linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/command/template/converter/Converter.java index 327f44cf2e..757d73d178 100644 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/utils/converter/Converter.java +++ b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/command/template/converter/Converter.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.linkis.cli.core.utils.converter; +package org.apache.linkis.cli.application.interactor.command.template.converter; public interface Converter { TO convert(FROM from); diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/utils/converter/PredefinedStringConverters.java b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/command/template/converter/PredefinedStringConverters.java similarity index 92% rename from linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/utils/converter/PredefinedStringConverters.java rename to linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/command/template/converter/PredefinedStringConverters.java index 5883f68e1b..b6e65c1e33 100644 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/utils/converter/PredefinedStringConverters.java +++ b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/command/template/converter/PredefinedStringConverters.java @@ -15,13 +15,13 @@ * limitations under the License. */ -package org.apache.linkis.cli.core.utils.converter; +package org.apache.linkis.cli.application.interactor.command.template.converter; -import org.apache.linkis.cli.common.exception.error.ErrorLevel; -import org.apache.linkis.cli.core.constants.CommonConstants; -import org.apache.linkis.cli.core.exception.CommandException; -import org.apache.linkis.cli.core.exception.error.CommonErrMsg; -import org.apache.linkis.cli.core.interactor.command.SpecialMap; +import org.apache.linkis.cli.application.constants.CliConstants; +import org.apache.linkis.cli.application.exception.CommandException; +import org.apache.linkis.cli.application.exception.error.CommonErrMsg; +import org.apache.linkis.cli.application.exception.error.ErrorLevel; +import org.apache.linkis.cli.application.interactor.command.SpecialMap; import org.apache.commons.lang3.StringUtils; @@ -50,7 +50,7 @@ public String[] convert(String from) { if (StringUtils.isBlank(from)) { return null; } - String[] ret = from.trim().split(CommonConstants.ARRAY_SEQ); + String[] ret = from.trim().split(CliConstants.ARRAY_SEQ); for (int i = 0; i < ret.length; i++) { ret[i] = StringUtils.strip(ret[i], " \""); } diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/command/template/option/BaseOption.java b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/command/template/option/BaseOption.java similarity index 93% rename from linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/command/template/option/BaseOption.java rename to linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/command/template/option/BaseOption.java index 8e9f288567..e497401bef 100644 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/command/template/option/BaseOption.java +++ b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/command/template/option/BaseOption.java @@ -15,10 +15,10 @@ * limitations under the License. */ -package org.apache.linkis.cli.core.interactor.command.template.option; +package org.apache.linkis.cli.application.interactor.command.template.option; -import org.apache.linkis.cli.common.entity.command.CmdOption; -import org.apache.linkis.cli.core.utils.converter.AbstractStringConverter; +import org.apache.linkis.cli.application.entity.command.CmdOption; +import org.apache.linkis.cli.application.interactor.command.template.converter.AbstractStringConverter; import org.apache.commons.lang3.StringUtils; diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/command/template/option/Flag.java b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/command/template/option/Flag.java similarity index 91% rename from linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/command/template/option/Flag.java rename to linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/command/template/option/Flag.java index 8b153fb7ac..ee66a64463 100644 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/command/template/option/Flag.java +++ b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/command/template/option/Flag.java @@ -15,9 +15,9 @@ * limitations under the License. */ -package org.apache.linkis.cli.core.interactor.command.template.option; +package org.apache.linkis.cli.application.interactor.command.template.option; -import org.apache.linkis.cli.core.utils.converter.PredefinedStringConverters; +import org.apache.linkis.cli.application.interactor.command.template.converter.PredefinedStringConverters; import org.apache.commons.lang3.StringUtils; diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/command/template/option/MapOption.java b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/command/template/option/MapOption.java similarity index 93% rename from linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/command/template/option/MapOption.java rename to linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/command/template/option/MapOption.java index 095004c4d1..d330d11c9b 100644 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/command/template/option/MapOption.java +++ b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/command/template/option/MapOption.java @@ -15,11 +15,11 @@ * limitations under the License. */ -package org.apache.linkis.cli.core.interactor.command.template.option; +package org.apache.linkis.cli.application.interactor.command.template.option; -import org.apache.linkis.cli.common.exception.error.ErrorLevel; -import org.apache.linkis.cli.core.exception.CommandException; -import org.apache.linkis.cli.core.exception.error.CommonErrMsg; +import org.apache.linkis.cli.application.exception.CommandException; +import org.apache.linkis.cli.application.exception.error.CommonErrMsg; +import org.apache.linkis.cli.application.exception.error.ErrorLevel; import org.apache.commons.lang3.StringUtils; diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/command/template/option/Parameter.java b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/command/template/option/Parameter.java similarity index 93% rename from linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/command/template/option/Parameter.java rename to linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/command/template/option/Parameter.java index 09422106c8..802f451ebb 100644 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/command/template/option/Parameter.java +++ b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/command/template/option/Parameter.java @@ -15,9 +15,9 @@ * limitations under the License. */ -package org.apache.linkis.cli.core.interactor.command.template.option; +package org.apache.linkis.cli.application.interactor.command.template.option; -import org.apache.linkis.cli.core.utils.converter.AbstractStringConverter; +import org.apache.linkis.cli.application.interactor.command.template.converter.AbstractStringConverter; import org.apache.commons.lang3.StringUtils; diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/command/template/option/SpecialMapOption.java b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/command/template/option/SpecialMapOption.java similarity index 90% rename from linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/command/template/option/SpecialMapOption.java rename to linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/command/template/option/SpecialMapOption.java index d14758206b..3fc6385db0 100644 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/command/template/option/SpecialMapOption.java +++ b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/command/template/option/SpecialMapOption.java @@ -15,9 +15,9 @@ * limitations under the License. */ -package org.apache.linkis.cli.core.interactor.command.template.option; +package org.apache.linkis.cli.application.interactor.command.template.option; -import org.apache.linkis.cli.core.interactor.command.SpecialMap; +import org.apache.linkis.cli.application.interactor.command.SpecialMap; /** kv-pairs in SpecialMapOption will be excluded by varAccess */ public class SpecialMapOption extends MapOption implements Cloneable { diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/command/template/option/StdOption.java b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/command/template/option/StdOption.java similarity index 92% rename from linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/command/template/option/StdOption.java rename to linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/command/template/option/StdOption.java index a9d29aef26..737cd73423 100644 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/command/template/option/StdOption.java +++ b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/command/template/option/StdOption.java @@ -15,9 +15,9 @@ * limitations under the License. */ -package org.apache.linkis.cli.core.interactor.command.template.option; +package org.apache.linkis.cli.application.interactor.command.template.option; -import org.apache.linkis.cli.core.utils.converter.AbstractStringConverter; +import org.apache.linkis.cli.application.interactor.command.template.converter.AbstractStringConverter; import org.apache.commons.lang3.StringUtils; diff --git a/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/context/CliCtxImpl.java b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/context/CliCtxImpl.java new file mode 100644 index 0000000000..7d4f5d94b8 --- /dev/null +++ b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/context/CliCtxImpl.java @@ -0,0 +1,60 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.cli.application.interactor.context; + +import org.apache.linkis.cli.application.entity.command.CmdTemplate; +import org.apache.linkis.cli.application.entity.command.CmdType; +import org.apache.linkis.cli.application.entity.context.CliCtx; +import org.apache.linkis.cli.application.entity.var.VarAccess; + +import java.util.Map; + +public class CliCtxImpl implements CliCtx { + private CmdType cmdType; + private CmdTemplate cmdTemplate; + private VarAccess varAccess; + private Map extraMap; + + public CliCtxImpl( + CmdType cmdType, CmdTemplate cmdTemplate, VarAccess varAccess, Map extraMap) { + this.cmdType = cmdType; + this.cmdTemplate = cmdTemplate; + this.varAccess = varAccess; + this.extraMap = extraMap; + } + + @Override + public CmdType getCmdType() { + return cmdType; + } + + @Override + public CmdTemplate getTemplate() { + return cmdTemplate; + } + + @Override + public VarAccess getVarAccess() { + return varAccess; + } + + @Override + public Map getExtraMap() { + return extraMap; + } +} diff --git a/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/job/common/JobKiller.java b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/job/common/JobKiller.java new file mode 100644 index 0000000000..6693abe85f --- /dev/null +++ b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/job/common/JobKiller.java @@ -0,0 +1,121 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.cli.application.interactor.job.common; + +import org.apache.linkis.cli.application.constants.CliConstants; +import org.apache.linkis.cli.application.operator.ujes.LinkisJobOper; +import org.apache.linkis.cli.application.operator.ujes.LinkisOperResultAdapter; +import org.apache.linkis.cli.application.utils.CliUtils; +import org.apache.linkis.cli.application.utils.LoggerManager; + +import org.apache.commons.lang3.exception.ExceptionUtils; + +import java.text.MessageFormat; +import java.util.HashMap; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class JobKiller { + private static final Logger logger = LoggerFactory.getLogger(JobKiller.class); + + private LinkisJobOper oper; + + public JobKiller(LinkisJobOper linkisJobOperator) { + this.oper = linkisJobOperator; + } + + public KillResult doKill(String username, String jobId) { + + LinkisOperResultAdapter jobInfoResult; + try { + jobInfoResult = oper.queryJobInfo(username, jobId); + } catch (Exception e) { + return new KillResult( + true, "Failed to query jobInfo" + ExceptionUtils.getStackTrace(e), new HashMap<>()); + } + if (jobInfoResult.getUser() == null || jobInfoResult.getJobID() == null) { + return new KillResult(false, "user or jobID is null", new HashMap<>()); + } + if (jobInfoResult.getJobStatus() == null) { + return new KillResult(false, "jobStatus is null", new HashMap<>()); + } + if (jobInfoResult.getJobStatus().isJobCancelled()) { + String msg = "Kill job aborted: Job has already been canceled."; + return new KillResult(false, msg, new HashMap<>()); + } else if (jobInfoResult.getJobStatus().isJobFinishedState()) { + String msg = "Kill job aborted: Job is already in finished-state(SUCCEED/FAILED)."; + return new KillResult(false, msg, new HashMap<>()); + // throw new LinkisClientExecutionException(JobStatus.FAILED, "EXE0004", + // ErrorLevel.ERROR, CommonErrMsg.ExecutionErr, msg); + } else { + try { + LinkisOperResultAdapter jobKillResult = + oper.kill( + jobInfoResult.getUser(), + jobInfoResult.getJobID(), + jobInfoResult.getStrongerExecId()); + } catch (Exception e) { + return new KillResult( + false, + "Exception thrown when trying to send kill request. Messgae: " + + ExceptionUtils.getStackTrace(e), + new HashMap<>()); + } + String msg = "Kill request has been sent"; + LoggerManager.getPlaintTextLogger().info(msg); + int retryCnt = 0; + final int MAX_RETRY = 5 * 6; + while (!jobInfoResult.getJobStatus().isJobFinishedState() + && !jobInfoResult.getJobStatus().isJobCancelled()) { + CliUtils.doSleepQuietly(CliConstants.JOB_QUERY_SLEEP_MILLS); + try { + jobInfoResult = oper.queryJobInfo(jobInfoResult.getUser(), jobInfoResult.getJobID()); + retryCnt = 0; // if exception then will not go here + } catch (Exception e) { + retryCnt++; + CliUtils.doSleepQuietly(5 * CliConstants.JOB_QUERY_SLEEP_MILLS); + if (retryCnt >= MAX_RETRY) { + return new KillResult( + false, + MessageFormat.format( + "After send kill. Client cannot get jobStatus from server continuously for {0} seconds. Client aborted. Assume kill failed! Error message: \n", + MAX_RETRY * 5 * CliConstants.JOB_QUERY_SLEEP_MILLS), + new HashMap<>()); + } + } + } + if (jobInfoResult.getJobStatus().isJobFinishedState() + && !jobInfoResult.getJobStatus().isJobCancelled()) { + msg = "Kill Failed: Job Current status: " + jobInfoResult.getJobStatus(); + return new KillResult(false, msg, new HashMap<>()); + // throw new LinkisClientExecutionException(JobStatus.FAILED, + // "EXE0004", ErrorLevel.ERROR, CommonErrMsg.ExecutionErr, msg); + } else if (jobInfoResult.getJobStatus().isJobCancelled()) { + msg = + MessageFormat.format( + "Kill successful: jobId={0}, status={1}.", + jobInfoResult.getJobID(), jobInfoResult.getJobStatus()); + return new KillResult(true, msg, new HashMap<>()); + // LogUtils.getPlaintTextLogger().info(msg); + } else { + return new KillResult(false, "Unknown Error!!", new HashMap<>()); + } + } + } +} diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/interactor/job/builder/ProcessKeyUtils.java b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/job/common/KeyParser.java similarity index 85% rename from linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/interactor/job/builder/ProcessKeyUtils.java rename to linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/job/common/KeyParser.java index 9527a8f513..01295ce676 100644 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/interactor/job/builder/ProcessKeyUtils.java +++ b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/job/common/KeyParser.java @@ -15,27 +15,27 @@ * limitations under the License. */ -package org.apache.linkis.cli.application.interactor.job.builder; +package org.apache.linkis.cli.application.interactor.job.common; -import org.apache.linkis.cli.application.constants.AppKeys; +import org.apache.linkis.cli.application.constants.CliKeys; import org.apache.commons.lang3.StringUtils; import java.util.HashMap; import java.util.Map; -public class ProcessKeyUtils { +public class KeyParser { public static Map removePrefixForKeysInMap(Map map) { final String[] PREFIX = new String[] { - AppKeys.JOB_PARAM_CONF, - AppKeys.JOB_PARAM_RUNTIME, - AppKeys.JOB_PARAM_VAR, - AppKeys.JOB_EXEC, - AppKeys.JOB_SOURCE, - AppKeys.JOB_LABEL, - AppKeys.JOB_CONTENT + CliKeys.JOB_PARAM_CONF, + CliKeys.JOB_PARAM_RUNTIME, + CliKeys.JOB_PARAM_VAR, + CliKeys.JOB_EXEC, + CliKeys.JOB_SOURCE, + CliKeys.JOB_LABEL, + CliKeys.JOB_CONTENT }; for (String prefix : PREFIX) { map = removePrefixForKeysInMap(map, prefix); diff --git a/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/job/common/KillResult.java b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/job/common/KillResult.java new file mode 100644 index 0000000000..732cd00ec7 --- /dev/null +++ b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/job/common/KillResult.java @@ -0,0 +1,61 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.cli.application.interactor.job.common; + +import org.apache.linkis.cli.application.entity.job.JobResult; + +import java.util.Map; + +public class KillResult implements JobResult { + private Boolean success; + private String message; + private Map extraMessage; + + public KillResult(Boolean success, String message, Map extraMessage) { + this.success = success; + this.message = message; + this.extraMessage = extraMessage; + } + + @Override + public Boolean isSuccess() { + return success; + } + + @Override + public String getMessage() { + return message; + } + + public void setMessage(String message) { + this.message = message; + } + + @Override + public Map getExtraMessage() { + return extraMessage; + } + + public void setExtraMessage(Map extraMessage) { + this.extraMessage = extraMessage; + } + + public void setSuccess(Boolean success) { + this.success = success; + } +} diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/interactor/job/LinkisJobStatus.java b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/job/common/LinkisJobStatus.java similarity index 97% rename from linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/interactor/job/LinkisJobStatus.java rename to linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/job/common/LinkisJobStatus.java index 9b1435da2a..dd4df9ee9b 100644 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/interactor/job/LinkisJobStatus.java +++ b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/job/common/LinkisJobStatus.java @@ -15,10 +15,10 @@ * limitations under the License. */ -package org.apache.linkis.cli.application.interactor.job; +package org.apache.linkis.cli.application.interactor.job.common; +import org.apache.linkis.cli.application.entity.job.JobStatus; import org.apache.linkis.cli.application.operator.once.LinkisNodeStatus; -import org.apache.linkis.cli.common.entity.job.JobStatus; import org.apache.commons.lang3.StringUtils; diff --git a/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/job/common/LogData.java b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/job/common/LogData.java new file mode 100644 index 0000000000..f9efe9531e --- /dev/null +++ b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/job/common/LogData.java @@ -0,0 +1,112 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.cli.application.interactor.job.common; + +import org.apache.linkis.cli.application.operator.ujes.LinkisOperResultAdapter; + +import java.util.LinkedList; +import java.util.List; +import java.util.concurrent.LinkedBlockingDeque; + +public class LogData { + private String user; + private String jobId; + private String execId; + private LinkedBlockingDeque logBuffer = new LinkedBlockingDeque(); + private String logPath; // remote path for job log + private Integer nextLogLineIdx; + private Boolean hasNextLogLine; + private volatile Boolean logFin = false; + + public LogData(String user, String jobId, String execId) { + this.user = user; + this.jobId = jobId; + this.execId = execId; + } + + public String getUser() { + return user; + } + + public String getJobID() { + return jobId; + } + + public String getExecID() { + return execId; + } + + public final String getLogPath() { + return logPath; + } + + public final void setLogPath(String logPath) { + this.logPath = logPath; + } + + public String consumeLog() { + List logs = new LinkedList<>(); + this.logBuffer.drainTo(logs, this.logBuffer.size()); + StringBuilder tmp = new StringBuilder(); + for (String str : logs) { + tmp.append(str); + } + return tmp.toString(); + } + + public void appendLog(String log) { + this.logBuffer.add(log); + } + + public Integer getNextLogLineIdx() { + return nextLogLineIdx; + } + + public void setNextLogLineIdx(Integer nextLogLineIdx) { + this.nextLogLineIdx = nextLogLineIdx; + } + + public Boolean hasNextLogLine() { + return hasNextLogLine; + } + + public void setHasNextLogLine(Boolean hasNextLogLine) { + this.hasNextLogLine = hasNextLogLine; + } + + public void updateLog(LinkisOperResultAdapter adapter) { + if (adapter.getLogPath() != null) { + setLogPath(adapter.getLogPath()); + } + if (adapter.getLog() != null + && adapter.getNextLogLine() != null + && adapter.hasNextLogLine() != null) { + setNextLogLineIdx(adapter.getNextLogLine()); + setHasNextLogLine(adapter.hasNextLogLine()); + appendLog(adapter.getLog()); + } + } + + public void setLogFin() { + this.logFin = true; + } + + public Boolean isLogFin() { + return logFin; + } +} diff --git a/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/job/common/LogRetriever.java b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/job/common/LogRetriever.java new file mode 100644 index 0000000000..a1dd5a86ac --- /dev/null +++ b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/job/common/LogRetriever.java @@ -0,0 +1,202 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.cli.application.interactor.job.common; + +import org.apache.linkis.cli.application.constants.CliConstants; +import org.apache.linkis.cli.application.exception.LinkisClientExecutionException; +import org.apache.linkis.cli.application.exception.LinkisClientRuntimeException; +import org.apache.linkis.cli.application.exception.error.CommonErrMsg; +import org.apache.linkis.cli.application.exception.error.ErrorLevel; +import org.apache.linkis.cli.application.observer.event.LinkisClientEvent; +import org.apache.linkis.cli.application.observer.event.LogStartEvent; +import org.apache.linkis.cli.application.observer.listener.LinkisClientListener; +import org.apache.linkis.cli.application.operator.ujes.LinkisJobOper; +import org.apache.linkis.cli.application.operator.ujes.LinkisOperResultAdapter; +import org.apache.linkis.cli.application.utils.CliUtils; +import org.apache.linkis.cli.application.utils.LoggerManager; +import org.apache.linkis.cli.application.utils.SchedulerManager; + +import java.text.MessageFormat; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class LogRetriever { + private static final Logger logger = LoggerFactory.getLogger(LogRetriever.class); + + private LinkisJobOper linkisJobOperator; + private LogData logData; + + private Boolean incLogMode; + + private LinkisClientListener logListener; + private LinkisClientEvent logStartEvent = new LogStartEvent(); + + public LogRetriever( + String user, + String jobId, + String execId, + Boolean incLogMode, + LinkisJobOper linkisJobOperator, + LinkisClientListener logListener) { + this.linkisJobOperator = linkisJobOperator; + this.logListener = logListener; + this.incLogMode = incLogMode; + this.logData = new LogData(user, jobId, execId); + registerLogListener(logListener); + } + + public void retrieveLogAsync() { + if (logData.getUser() == null || logData.getJobID() == null) { + throw new LinkisClientExecutionException( + "EXE0036", ErrorLevel.ERROR, CommonErrMsg.ExecutionErr, "user or jobID is null"); + } + try { + Thread logConsumer = new Thread(() -> notifyLogListener(), "Log-Consumer"); + Thread logRetriever = new Thread(() -> queryLogLoop(logData), "Log-Retriever"); + SchedulerManager.getCachedThreadPoolExecutor().execute(logRetriever); + SchedulerManager.getCachedThreadPoolExecutor().execute(logConsumer); + } catch (Exception e) { + logger.warn("Failed to retrieve log", e); + } + } + + public void waitIncLogComplete() { + int retry = 0; + int MAX_RETRY = 300; // wait for 10 minutes after job finish + while (retry++ < MAX_RETRY) { + if (this.logFinReceived()) { + return; + } + CliUtils.doSleepQuietly(CliConstants.JOB_QUERY_SLEEP_MILLS); + } + String msg = + "Job is in Finished state(SUCCEED/FAILED/CANCELED) but client keep querying inclog for " + + (MAX_RETRY * CliConstants.JOB_QUERY_SLEEP_MILLS / 1000) + + "seconds. Execution ends forcefully. Next will try handle execution result."; + logger.warn(msg); + LoggerManager.getInformationLogger().warn(msg); + } + + public void queryLogLoop(LogData data) { + int curLogIdx; + int nextLogIdx; + boolean hasNext = true; + int retryCnt = 0; + final int MAX_RETRY = 5; + try { + while (hasNext) { + curLogIdx = data.getNextLogLineIdx() == null ? 0 : data.getNextLogLineIdx(); + try { + queryJobLogFromLine(data, curLogIdx); + } catch (Exception e) { + logger.error("Cannot get inc-log:", e); + // and yes sometimes server may not be able to prepare persisted-log + retryCnt++; + if (retryCnt >= MAX_RETRY) { + logger.error( + "Continuously failing to query inc-log for " + + MAX_RETRY * (MAX_RETRY + 2) * 500 / 1000 + + "s. Will no longer try to query log", + e); + break; + } + CliUtils.doSleepQuietly(500l + 500l * retryCnt); // maybe server problem. sleep longer + continue; + } + retryCnt = 0; + nextLogIdx = data.getNextLogLineIdx() == null ? curLogIdx : data.getNextLogLineIdx(); + if (incLogMode) { + hasNext = data.hasNextLogLine() == null ? curLogIdx < nextLogIdx : data.hasNextLogLine(); + } else { + hasNext = curLogIdx < nextLogIdx; + } + if (curLogIdx >= nextLogIdx) { + String msg = + MessageFormat.format( + "Retrieving log, hasNext={0}, nextLogIdx={1}", hasNext, nextLogIdx); + logger.info(msg); + } + CliUtils.doSleepQuietly(CliConstants.JOB_QUERY_SLEEP_MILLS); + } + } catch (Exception e) { + logger.error("Something goes wrong. Job Log may be incomplete", e); + } finally { + sendLogFin(); + } + } + + private void queryJobLogFromLine(LogData data, int fromLine) throws LinkisClientRuntimeException { + + LinkisOperResultAdapter jobInfoResult = + linkisJobOperator.queryJobInfo(data.getUser(), data.getJobID()); + data.updateLog(jobInfoResult); + if (!jobInfoResult.getJobStatus().isJobFinishedState()) { + try { + data.updateLog( + linkisJobOperator.queryRunTimeLogFromLine( + data.getUser(), data.getJobID(), data.getExecID(), fromLine)); + } catch (Exception e) { + // job is finished while we start query log(but request is not send). + // then probably server cache is gone and we got a exception here. + // however we cannot know if this happens based on the exception message + logger.warn( + "Caught exception when querying runtime-log. Probably server-side has close stream. Will try openLog api if Job is completed.", + e); + if (jobInfoResult.getJobStatus().isJobFinishedState()) { + CliUtils.doSleepQuietly(500l); + data.updateLog( + linkisJobOperator.queryPersistedLogFromLine( + data.getLogPath(), data.getUser(), data.getJobID(), fromLine)); + } + } + } else { + try { + data.updateLog( + linkisJobOperator.queryPersistedLogFromLine( + data.getLogPath(), data.getUser(), data.getJobID(), fromLine)); + } catch (Exception e) { + logger.error("Cannot get persisted-inc-log:", e); + // and yes sometimes server may not be able to prepare persisted-log + throw e; + } + } + } + + public Boolean isIncLogMode() { + return incLogMode; + } + + public void registerLogListener(LinkisClientListener observer) { + this.logStartEvent.register(observer); + } + + public void notifyLogListener() { + if (this.logStartEvent.isRegistered()) { + logStartEvent.notifyObserver(logStartEvent, this.logData); + } + } + + public void sendLogFin() { + this.logData.setLogFin(); + } + + public boolean logFinReceived() { + return this.logData.isLogFin(); + } +} diff --git a/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/job/common/ResultData.java b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/job/common/ResultData.java new file mode 100644 index 0000000000..8c9bec028b --- /dev/null +++ b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/job/common/ResultData.java @@ -0,0 +1,165 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.cli.application.interactor.job.common; + +import org.apache.linkis.cli.application.entity.job.JobStatus; +import org.apache.linkis.cli.application.operator.ujes.LinkisOperResultAdapter; + +import java.util.LinkedList; +import java.util.List; +import java.util.concurrent.LinkedBlockingDeque; + +public class ResultData { + + private final String user; + private final String jobId; + private final String execId; + private final LinkedBlockingDeque resultContent = new LinkedBlockingDeque<>(); + private String extraMessage; + private JobStatus jobStatus = null; + private String resultLocation; + private String[] resultSetPaths = null; // remote paths for job result set + private Boolean hasNextResultPage; + private Integer errCode = null; + private String errDesc = null; + private boolean hasResult = true; + + private volatile Boolean resultFin = false; + + public ResultData(String user, String jobId, String execId) { + this.user = user; + this.jobId = jobId; + this.execId = execId; + } + + public String getJobID() { + return jobId; + } + + public String getUser() { + return user; + } + + public final String getExecID() { + return execId; + } + + public final String getResultLocation() { + return resultLocation; + } + + public final void setResultLocation(String resultLocation) { + this.resultLocation = resultLocation; + } + + public String[] getResultSetPaths() { + return resultSetPaths; + } + + public final void setResultSetPaths(String[] resultSetPaths) { + this.resultSetPaths = resultSetPaths; + } + + public Integer getErrCode() { + return errCode; + } + + public void setErrCode(Integer errCode) { + this.errCode = errCode; + } + + public String getErrDesc() { + return errDesc; + } + + public void setErrDesc(String errDesc) { + this.errDesc = errDesc; + } + + public List consumeResultContent() { + List ret = new LinkedList<>(); + resultContent.drainTo(ret, resultContent.size()); + return ret; + } + + public void appendResultContent(ResultSet resultContent) { + this.resultContent.add(resultContent); + } + + public Boolean hasNextResultPage() { + return hasNextResultPage; + } + + public void setHasNextResultPage(Boolean hasNextResultPage) { + this.hasNextResultPage = hasNextResultPage; + } + + public void setResultFin() { + this.resultFin = true; + } + + public boolean isResultFin() { + return this.resultFin; + } + + public boolean hasResult() { + return hasResult; + } + + public void setHasResult(boolean hasResult) { + this.hasResult = hasResult; + } + + public JobStatus getJobStatus() { + return jobStatus; + } + + public void setJobStatus(JobStatus jobStatus) { + this.jobStatus = jobStatus; + } + + public String getExtraMessage() { + return extraMessage; + } + + public void setExtraMessage(String extraMessage) { + this.extraMessage = extraMessage; + } + + public void updateByOperResult(LinkisOperResultAdapter adapter) { + if (adapter.getResultLocation() != null) { + setResultLocation(adapter.getResultLocation()); + } + if (adapter.getResultSetPaths() != null) { + setResultSetPaths(adapter.getResultSetPaths()); + } + if (adapter.getErrCode() != null) { + setErrCode(adapter.getErrCode()); + } + if (adapter.getErrDesc() != null) { + setErrDesc(adapter.getErrDesc()); + } + if (adapter.getResultContent() != null && adapter.resultHasNextPage() != null) { + setHasNextResultPage(adapter.resultHasNextPage()); + appendResultContent(adapter.getResultContent()); + } + if (adapter.getJobStatus() != null) { + setJobStatus(adapter.getJobStatus()); + } + } +} diff --git a/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/job/common/ResultRetriever.java b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/job/common/ResultRetriever.java new file mode 100644 index 0000000000..9997dd1f41 --- /dev/null +++ b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/job/common/ResultRetriever.java @@ -0,0 +1,186 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.cli.application.interactor.job.common; + +import org.apache.linkis.cli.application.constants.CliConstants; +import org.apache.linkis.cli.application.exception.LinkisClientExecutionException; +import org.apache.linkis.cli.application.exception.LinkisClientRuntimeException; +import org.apache.linkis.cli.application.exception.error.CommonErrMsg; +import org.apache.linkis.cli.application.exception.error.ErrorLevel; +import org.apache.linkis.cli.application.observer.event.FetchResultEvent; +import org.apache.linkis.cli.application.observer.event.LinkisClientEvent; +import org.apache.linkis.cli.application.observer.listener.LinkisClientListener; +import org.apache.linkis.cli.application.operator.ujes.LinkisJobOper; +import org.apache.linkis.cli.application.utils.CliUtils; +import org.apache.linkis.cli.application.utils.LoggerManager; +import org.apache.linkis.cli.application.utils.SchedulerManager; + +import org.apache.commons.lang3.StringUtils; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class ResultRetriever { + private static final Logger logger = LoggerFactory.getLogger(ResultRetriever.class); + + private LinkisJobOper linkisJobOperator; + private ResultData resultData; + + private LinkisClientListener resultListener; + private LinkisClientEvent fetchResultEvent = new FetchResultEvent(); + + public ResultRetriever( + String user, + String jobId, + String execId, + LinkisJobOper linkisJobOperator, + LinkisClientListener resultListener) { + this.linkisJobOperator = linkisJobOperator; + this.resultListener = resultListener; + this.resultData = new ResultData(user, jobId, execId); + registerResultListener(resultListener); + } + + public void retrieveResultSync() { + if (resultData.getUser() == null || resultData.getJobID() == null) { + throw new LinkisClientExecutionException( + "EXE0036", ErrorLevel.ERROR, CommonErrMsg.ExecutionErr, "user or jobID is null"); + } + resultData.updateByOperResult( + linkisJobOperator.queryJobInfo(resultData.getUser(), resultData.getJobID())); + if (resultData.getJobStatus() == null) { + throw new LinkisClientExecutionException( + "EXE0038", ErrorLevel.ERROR, CommonErrMsg.ExecutionErr, "jobStatus is null"); + } + if (!resultData.getJobStatus().isJobSuccess()) { + LoggerManager.getInformationLogger() + .info( + "Job status is not success but \'" + + resultData.getJobStatus() + + "\'. Will not try to retrieve any Result"); + resultData.setResultFin(); // inform listener to stop + return; + } + if (StringUtils.isBlank(resultData.getResultLocation())) { + throw new LinkisClientExecutionException( + "EXE0037", + ErrorLevel.WARN, + CommonErrMsg.ExecutionErr, + "Got blank ResultLocation from server. Job may not have result-set. Will not try to retrieve any Result"); + } + resultData.updateByOperResult( + linkisJobOperator.queryResultSetPaths( + resultData.getUser(), resultData.getJobID(), resultData.getResultLocation())); + + if (resultData.getResultSetPaths() == null || resultData.getResultSetPaths().length == 0) { + String msg = "Your job got no result."; + logger.warn(msg); + resultData.setResultFin(); // inform listener to stop + resultData.setHasResult(false); + return; + } + + try { + resultData.setHasResult(true); + // Thread resultConsumer = new Thread(() -> notifyResultListener()); + Thread resultThread = new Thread(() -> queryResultLoop(resultData), "Result-Retrieve-Thread"); + // SchedulerUtils.getCachedThreadPoolExecutor().execute(resultConsumer); + SchedulerManager.getCachedThreadPoolExecutor().execute(resultThread); + notifyResultListener(); + } catch (Exception e) { + logger.error("Failed to retrieve result", e); + throw e; + } + } + + public void queryResultLoop(ResultData data) { + boolean hasNext = true; + int retryCnt = 0; + final int MAX_RETRY = 30; // continues fails for 250s, then exit + int idx = 0; + try { + while (hasNext) { + try { + hasNext = queryOneResult(data, idx); + } catch (LinkisClientRuntimeException e) { + logger.error("Cannot get result:", e); + retryCnt++; + if (retryCnt >= MAX_RETRY) { + logger.error( + "Continuously failing to query result for " + + MAX_RETRY * (MAX_RETRY + 2) * 500 / 1000 + + "s. Will no longer try to query result", + e); + return; + } else { + hasNext = true; + } + CliUtils.doSleepQuietly(500l + 500l * retryCnt); // maybe server problem. sleep longer + continue; + } + idx++; + } + } catch (Exception e) { + logger.error("Something goes wrong. Job Result may be incomplete", e); + throw e; + } finally { + data.setResultFin(); + } + } + + private boolean queryOneResult(ResultData data, int idxResultSet) { + Integer curPage = 1; + boolean hasNextResult = true; + boolean hasNextPage = true; + while (hasNextPage) { + data.updateByOperResult( + linkisJobOperator.queryResultSetGivenResultSetPath( + data.getResultSetPaths(), + idxResultSet, + data.getUser(), + curPage, + CliConstants.RESULTSET_PAGE_SIZE)); + if (data.hasNextResultPage() == null) { + throw new LinkisClientExecutionException( + "EXE0040", + ErrorLevel.ERROR, + CommonErrMsg.ExecutionResultErr, + "Something foes wrong. Got null as \'hasNextPage\'."); + } + hasNextPage = data.hasNextResultPage(); + + curPage++; + hasNextResult = idxResultSet + 1 < data.getResultSetPaths().length; + } + return hasNextResult; + } + + public void registerResultListener(LinkisClientListener observer) { + this.fetchResultEvent.register(observer); + } + + public void notifyResultListener() { + if (this.fetchResultEvent.isRegistered()) { + fetchResultEvent.notifyObserver(fetchResultEvent, this.resultData); + } + } + + public void setResultFin() { + this.resultData.setResultFin(); + } +} diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/interactor/job/data/LinkisResultSet.java b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/job/common/ResultSet.java similarity index 88% rename from linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/interactor/job/data/LinkisResultSet.java rename to linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/job/common/ResultSet.java index 4fa1dcfe02..5821e14a55 100644 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/interactor/job/data/LinkisResultSet.java +++ b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/job/common/ResultSet.java @@ -15,18 +15,18 @@ * limitations under the License. */ -package org.apache.linkis.cli.application.interactor.job.data; +package org.apache.linkis.cli.application.interactor.job.common; import java.util.LinkedHashMap; import java.util.LinkedList; import java.util.List; -public class LinkisResultSet implements Cloneable { +public class ResultSet implements Cloneable { private int resultsetIdx; private List> resultMeta; private List> content; - public LinkisResultSet() {} + public ResultSet() {} public int getResultsetIdx() { return resultsetIdx; @@ -53,8 +53,8 @@ public void setContent(List> content) { } @Override - protected LinkisResultSet clone() throws CloneNotSupportedException { - LinkisResultSet ret = new LinkisResultSet(); + protected ResultSet clone() throws CloneNotSupportedException { + ResultSet ret = new ResultSet(); if (this.resultMeta != null) { List> resultMeta = null; ret.resultMeta = new LinkedList<>(); diff --git a/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/job/help/HelpJob.java b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/job/help/HelpJob.java new file mode 100644 index 0000000000..943c54ed63 --- /dev/null +++ b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/job/help/HelpJob.java @@ -0,0 +1,62 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.cli.application.interactor.job.help; + +import org.apache.linkis.cli.application.entity.context.CliCtx; +import org.apache.linkis.cli.application.entity.job.Job; +import org.apache.linkis.cli.application.entity.job.JobResult; +import org.apache.linkis.cli.application.present.HelpPresenter; +import org.apache.linkis.cli.application.present.model.HelpInfoModel; + +import java.util.HashMap; +import java.util.Map; + +public class HelpJob implements Job { + private CliCtx ctx; + + @Override + public void build(CliCtx ctx) { + this.ctx = ctx; + } + + @Override + public JobResult run() { + HelpInfoModel model = new HelpInfoModel(); + model.buildModel(ctx.getTemplate()); + new HelpPresenter().present(model); + return new JobResult() { + @Override + public Boolean isSuccess() { + return true; + } + + @Override + public String getMessage() { + return ""; + } + + @Override + public Map getExtraMessage() { + return new HashMap<>(); + } + }; + } + + @Override + public void onDestroy() {} +} diff --git a/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/job/interactive/InteractiveJob.java b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/job/interactive/InteractiveJob.java new file mode 100644 index 0000000000..f1b8f00c68 --- /dev/null +++ b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/job/interactive/InteractiveJob.java @@ -0,0 +1,260 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.cli.application.interactor.job.interactive; + +import org.apache.linkis.cli.application.constants.CliConstants; +import org.apache.linkis.cli.application.constants.CliKeys; +import org.apache.linkis.cli.application.entity.context.CliCtx; +import org.apache.linkis.cli.application.entity.job.Job; +import org.apache.linkis.cli.application.entity.job.JobResult; +import org.apache.linkis.cli.application.exception.LinkisClientExecutionException; +import org.apache.linkis.cli.application.exception.LinkisClientRuntimeException; +import org.apache.linkis.cli.application.exception.error.CommonErrMsg; +import org.apache.linkis.cli.application.exception.error.ErrorLevel; +import org.apache.linkis.cli.application.interactor.job.common.JobKiller; +import org.apache.linkis.cli.application.interactor.job.common.LogRetriever; +import org.apache.linkis.cli.application.interactor.job.common.ResultRetriever; +import org.apache.linkis.cli.application.operator.OperManager; +import org.apache.linkis.cli.application.operator.ujes.LinkisJobOper; +import org.apache.linkis.cli.application.operator.ujes.LinkisOperResultAdapter; +import org.apache.linkis.cli.application.present.LogPresenter; +import org.apache.linkis.cli.application.present.ResultPresenter; +import org.apache.linkis.cli.application.utils.CliUtils; +import org.apache.linkis.cli.application.utils.LoggerManager; + +import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.exception.ExceptionUtils; + +import java.util.HashMap; +import java.util.Map; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class InteractiveJob implements Job { + + private static final Logger logger = LoggerFactory.getLogger(InteractiveJob.class); + + private CliCtx ctx; + + private Boolean isAsync = false; + + private LinkisJobOper oper; + + private InteractiveJobDesc desc; + + private String username; + + private String jobId; + + @Override + public void build(CliCtx ctx) { + this.ctx = ctx; + this.isAsync = + ctx.getVarAccess().getVarOrDefault(Boolean.class, CliKeys.LINKIS_CLIENT_ASYNC_OPT, false); + this.desc = InteractiveJobDescBuilder.build(ctx); + this.oper = (LinkisJobOper) OperManager.getNew(CliKeys.Linkis_OPER, ctx); + } + + @Override + public JobResult run() { + + // Indicator + StringBuilder infoBuilder = new StringBuilder(); + infoBuilder.append("connecting to linkis gateway:").append(oper.getServerUrl()); + LoggerManager.getInformationLogger().info(infoBuilder.toString()); + infoBuilder.setLength(0); + + // Submit + LinkisOperResultAdapter submitResult = oper.submit(desc); + CliUtils.doSleepQuietly(CliConstants.JOB_QUERY_SLEEP_MILLS); + + // JobInfo + LinkisOperResultAdapter jobInfoResult = + oper.queryJobInfo(submitResult.getUser(), submitResult.getJobID()); + infoBuilder.setLength(0); + infoBuilder + .append("JobId:") + .append(jobInfoResult.getJobID()) + .append(System.lineSeparator()) + .append("TaskId:") + .append(jobInfoResult.getJobID()) + .append(System.lineSeparator()) + .append("ExecId:") + .append(jobInfoResult.getStrongerExecId()); + LoggerManager.getPlaintTextLogger().info(infoBuilder.toString()); + infoBuilder.setLength(0); + + // Submit success or not + if (!jobInfoResult.getJobStatus().isJobSubmitted()) { + return new InteractiveJobResult(false, "Failed to submit job", new HashMap<>()); + } else { + // Output that job is submitted + infoBuilder.append("Job is successfully submitted!").append(System.lineSeparator()); + LoggerManager.getInformationLogger().info(infoBuilder.toString()); + infoBuilder.setLength(0); + username = submitResult.getUser(); + jobId = submitResult.getJobID(); + } + + // async job, return + if (isAsync) { + return new InteractiveJobResult( + submitResult.getJobStatus().isJobSubmitted(), + "Async Submission Success", + new HashMap<>()); + } + + CliUtils.doSleepQuietly(2000l); + + // get log while running + LogRetriever logRetriever = + new LogRetriever( + jobInfoResult.getUser(), + jobInfoResult.getJobID(), + jobInfoResult.getStrongerExecId(), + true, + oper, + new LogPresenter()); + // async because we need to query job status + logRetriever.retrieveLogAsync(); + + // wait complete + jobInfoResult = waitJobComplete(submitResult.getUser(), submitResult.getJobID()); + logRetriever.waitIncLogComplete(); + + // get result-set + String outputPath = + ctx.getVarAccess().getVar(String.class, CliKeys.LINKIS_CLIENT_COMMON_OUTPUT_PATH); + ResultPresenter presenter; + if (StringUtils.isBlank(outputPath)) { + presenter = new ResultPresenter(); + } else { + presenter = new ResultPresenter(true, outputPath); + } + + ResultRetriever resultRetriever = + new ResultRetriever( + jobInfoResult.getUser(), + jobInfoResult.getJobID(), + jobInfoResult.getStrongerExecId(), + oper, + presenter); + + JobResult result = getResult(jobInfoResult, resultRetriever); + + return result; + } + + private JobResult getResult( + LinkisOperResultAdapter jobInfoResult, ResultRetriever resultRetriever) + throws LinkisClientRuntimeException { + if (!jobInfoResult.getJobStatus().isJobSuccess()) { + LoggerManager.getInformationLogger() + .info( + "Job status is not success but \'" + + jobInfoResult.getJobStatus() + + "\'. Will not try to retrieve any Result"); + Map extraMap = new HashMap<>(); + if (jobInfoResult.getErrCode() != null) { + extraMap.put("errorCode", String.valueOf(jobInfoResult.getErrCode())); + } + if (StringUtils.isNotBlank(jobInfoResult.getErrDesc())) { + extraMap.put("errorDesc", jobInfoResult.getErrDesc()); + } + return new InteractiveJobResult(false, "Execute Error!!!", extraMap); + } + InteractiveJobResult result = + new InteractiveJobResult(true, "Execute Success!!!", new HashMap<>()); + try { + resultRetriever.retrieveResultSync(); + result.setSuccess(true); + result.setMessage("execute success!!!"); + } catch (LinkisClientExecutionException e) { + if (e.getCode().equals("EXE0037")) { + result.setSuccess(true); + result.setMessage("execute success!!!"); + LoggerManager.getInformationLogger().warn(e.getMessage()); + } else { + result.setSuccess(false); + result.setMessage("execute failed!!!\n" + ExceptionUtils.getStackTrace(e)); + } + resultRetriever.setResultFin(); // inform listener to stop + } catch (Exception e) { + result.setSuccess(false); + result.setMessage("execute failed!!!\n" + ExceptionUtils.getStackTrace(e)); + resultRetriever.setResultFin(); // inform listener to stop + } + return result; + } + + private LinkisOperResultAdapter waitJobComplete(String user, String jobId) + throws LinkisClientRuntimeException { + int retryCnt = 0; + final int MAX_RETRY = 30; + + LinkisOperResultAdapter jobInfoResult = oper.queryJobInfo(user, jobId); + oper.queryJobStatus(user, jobId, jobInfoResult.getStrongerExecId()); + + while (!jobInfoResult.getJobStatus().isJobFinishedState()) { + // query progress + try { + jobInfoResult = oper.queryJobInfo(user, jobId); + } catch (Exception e) { + logger.warn("", e); + retryCnt++; + if (retryCnt >= MAX_RETRY) { + throw new LinkisClientExecutionException( + "EXE0013", + ErrorLevel.ERROR, + CommonErrMsg.ExecutionErr, + "Cannot get jobStatus from server continuously for {0} seconds. Client aborted! Error message: \n", + MAX_RETRY * 5 * CliConstants.JOB_QUERY_SLEEP_MILLS, + e); + } + CliUtils.doSleepQuietly( + 5 * CliConstants.JOB_QUERY_SLEEP_MILLS); // maybe server problem. sleep + // longer + continue; + } + retryCnt = 0; // reset counter + if (jobInfoResult.getJobStatus().isJobAbnormalStatus()) { + throw new LinkisClientExecutionException( + "EXE0006", + ErrorLevel.ERROR, + CommonErrMsg.ExecutionErr, + "Job is in abnormal status: " + CliUtils.GSON.toJson(jobInfoResult)); + } + CliUtils.doSleepQuietly(CliConstants.JOB_QUERY_SLEEP_MILLS); + } + return jobInfoResult; + } + + @Override + public void onDestroy() { + if (StringUtils.isBlank(username) || StringUtils.isBlank(jobId)) { + logger.warn("Failed to kill job username or jobId is blank"); + return; + } + try { + new JobKiller(oper).doKill(username, jobId); + } catch (Exception e) { + logger.error("Failed to kill job", e); + } + } +} diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/interactor/job/desc/LinkisSubmitDesc.java b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/job/interactive/InteractiveJobDesc.java similarity index 93% rename from linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/interactor/job/desc/LinkisSubmitDesc.java rename to linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/job/interactive/InteractiveJobDesc.java index 49cdd1a547..e594d9cc23 100644 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/interactor/job/desc/LinkisSubmitDesc.java +++ b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/job/interactive/InteractiveJobDesc.java @@ -15,13 +15,11 @@ * limitations under the License. */ -package org.apache.linkis.cli.application.interactor.job.desc; - -import org.apache.linkis.cli.common.entity.job.JobDescription; +package org.apache.linkis.cli.application.interactor.job.interactive; import java.util.Map; -public class LinkisSubmitDesc implements JobDescription { +public class InteractiveJobDesc { private String submitUser; private String proxyUser; private String creator; diff --git a/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/job/interactive/InteractiveJobDescBuilder.java b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/job/interactive/InteractiveJobDescBuilder.java new file mode 100644 index 0000000000..2b0b20188a --- /dev/null +++ b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/job/interactive/InteractiveJobDescBuilder.java @@ -0,0 +1,177 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.cli.application.interactor.job.interactive; + +import org.apache.linkis.cli.application.constants.CliConstants; +import org.apache.linkis.cli.application.constants.CliKeys; +import org.apache.linkis.cli.application.constants.LinkisKeys; +import org.apache.linkis.cli.application.entity.context.CliCtx; +import org.apache.linkis.cli.application.entity.var.VarAccess; +import org.apache.linkis.cli.application.interactor.job.common.KeyParser; +import org.apache.linkis.cli.application.operator.ujes.LinkisJobOper; +import org.apache.linkis.cli.application.operator.ujes.UJESClientFactory; +import org.apache.linkis.cli.application.utils.CliUtils; +import org.apache.linkis.cli.application.utils.LoggerManager; + +import org.apache.commons.lang3.StringUtils; + +import java.util.*; + +public class InteractiveJobDescBuilder { + + public static InteractiveJobDesc build(CliCtx ctx) { + InteractiveJobDesc desc = new InteractiveJobDesc(); + + VarAccess stdVarAccess = ctx.getVarAccess(); + + Map confMap = stdVarAccess.getVar(Map.class, CliKeys.JOB_PARAM_CONF); + Map runtimeMap = stdVarAccess.getVar(Map.class, CliKeys.JOB_PARAM_RUNTIME); + Map varMap = stdVarAccess.getVar(Map.class, CliKeys.JOB_PARAM_VAR); + Map labelMap = stdVarAccess.getVar(Map.class, CliKeys.JOB_LABEL); + Map sourceMap = stdVarAccess.getVar(Map.class, CliKeys.JOB_SOURCE); + Map executionMap = stdVarAccess.getVar(Map.class, CliKeys.JOB_EXEC); + + confMap = confMap == null ? new HashMap<>() : confMap; + runtimeMap = runtimeMap == null ? new HashMap<>() : runtimeMap; + varMap = varMap == null ? new HashMap<>() : varMap; + labelMap = labelMap == null ? new HashMap<>() : labelMap; + sourceMap = sourceMap == null ? new HashMap<>() : sourceMap; + executionMap = executionMap == null ? new HashMap<>() : executionMap; + + /** remove key prefix of all keys in map type params. e.g. kv in confMap, labelMap etc. */ + confMap = KeyParser.removePrefixForKeysInMap(confMap); + runtimeMap = KeyParser.removePrefixForKeysInMap(runtimeMap); + labelMap = KeyParser.removePrefixForKeysInMap(labelMap); + sourceMap = KeyParser.removePrefixForKeysInMap(sourceMap); + executionMap = KeyParser.removePrefixForKeysInMap(executionMap); + + /** remove key prefix of non-map type params */ + for (String key : stdVarAccess.getAllVarKeys()) { + Object val = stdVarAccess.getVar(Object.class, key); + if (!(val instanceof Map) && val != null) { + // note that we allow it to overwrite existing values in map + if (StringUtils.startsWithIgnoreCase(key, CliKeys.JOB_PARAM_CONF)) { + KeyParser.removePrefixAndPutValToMap(confMap, key, val, CliKeys.JOB_PARAM_CONF); + } else if (StringUtils.startsWithIgnoreCase(key, CliKeys.JOB_PARAM_VAR)) { + KeyParser.removePrefixAndPutValToMap(varMap, key, val, CliKeys.JOB_PARAM_VAR); + } else if (StringUtils.startsWithIgnoreCase(key, CliKeys.JOB_PARAM_RUNTIME)) { + KeyParser.removePrefixAndPutValToMap(runtimeMap, key, val, CliKeys.JOB_PARAM_RUNTIME); + } else if (StringUtils.startsWithIgnoreCase(key, CliKeys.JOB_EXEC)) { + KeyParser.removePrefixAndPutValToMap(executionMap, key, val, CliKeys.JOB_EXEC); + } else if (StringUtils.startsWithIgnoreCase(key, CliKeys.JOB_LABEL)) { + KeyParser.removePrefixAndPutValToMap(labelMap, key, val, CliKeys.JOB_LABEL); + } else if (StringUtils.startsWithIgnoreCase(key, CliKeys.JOB_SOURCE)) { + KeyParser.removePrefixAndPutValToMap(sourceMap, key, val, CliKeys.JOB_SOURCE); + } else if (StringUtils.startsWithIgnoreCase(key, CliKeys.LINKIS_CLIENT_COMMON)) { + // do nothing + } else { + // confMap.put(key, stdVarAccess.getVar(Object.class, key)); + } + } + } + + Boolean asyncSubmission = + stdVarAccess.getVarOrDefault(Boolean.class, CliKeys.LINKIS_CLIENT_ASYNC_OPT, false); + + String creator; + if (!asyncSubmission) { + creator = + stdVarAccess.getVarOrDefault( + String.class, CliKeys.JOB_COMMON_CREATOR, CliConstants.JOB_CREATOR_DEFAULT); + } else { + creator = + stdVarAccess.getVarOrDefault( + String.class, CliKeys.JOB_COMMON_CREATOR, CliConstants.JOB_CREATOR_ASYNC_DEFAULT); + } + String code = stdVarAccess.getVar(String.class, CliKeys.JOB_EXEC_CODE); + String engineType = stdVarAccess.getVar(String.class, CliKeys.JOB_LABEL_ENGINE_TYPE); + String runType = stdVarAccess.getVar(String.class, CliKeys.JOB_LABEL_CODE_TYPE); + String scriptPath = + stdVarAccess.getVarOrDefault(String.class, CliKeys.JOB_SOURCE_SCRIPT_PATH, "LinkisCli"); + + String osUser = System.getProperty(CliKeys.LINUX_USER_KEY); + String[] adminUsers = StringUtils.split(CliKeys.ADMIN_USERS, ','); + Set adminSet = new HashSet<>(); + for (String admin : adminUsers) { + adminSet.add(admin); + } + String submitUsr = CliUtils.getSubmitUser(stdVarAccess, osUser, adminSet); + String proxyUsr = CliUtils.getProxyUser(stdVarAccess, submitUsr, adminSet); + + String enableExecuteOnce = + stdVarAccess.getVarOrDefault(String.class, CliKeys.JOB_LABEL_EXECUTEONCE, "true"); + // default executeOnce-mode + if (Boolean.parseBoolean(enableExecuteOnce)) { + labelMap.put(LinkisKeys.KEY_EXECUTEONCE, ""); + } else { + labelMap.remove(LinkisKeys.KEY_EXECUTEONCE); + } + String codePath = stdVarAccess.getVar(String.class, CliKeys.JOB_COMMON_CODE_PATH); + Object extraArgsObj = stdVarAccess.getVar(Object.class, CliKeys.JOB_EXTRA_ARGUMENTS); + if (extraArgsObj != null + && extraArgsObj instanceof String[] + && StringUtils.isBlank(code) + && StringUtils.isBlank(codePath)) { + String[] extraArgs = (String[]) extraArgsObj; + codePath = extraArgs[0]; + if (extraArgs.length > 1) { + runtimeMap.put( + LinkisKeys.EXTRA_ARGUMENTS, Arrays.copyOfRange(extraArgs, 1, extraArgs.length)); + } + } + + if (StringUtils.isBlank(code) && StringUtils.isNotBlank(codePath)) { + try { + code = CliUtils.readFile(codePath); + } catch (Exception e) { + LoggerManager.getInformationLogger().error("Failed to read file", e); + throw e; + } + } + + executionMap.put(LinkisKeys.KEY_CODE, code); + labelMap.put(LinkisKeys.KEY_ENGINETYPE, engineType); + labelMap.put(LinkisKeys.KEY_CODETYPE, runType); + labelMap.put(LinkisKeys.KEY_USER_CREATOR, proxyUsr + "-" + creator); + sourceMap.put(LinkisKeys.KEY_SCRIPT_PATH, scriptPath); + if (ctx.getExtraMap().containsKey(CliKeys.VERSION)) { + sourceMap.put(LinkisKeys.CLI_VERSION, ctx.getExtraMap().get(CliKeys.VERSION)); + } + runtimeMap.put(LinkisKeys.KEY_HIVE_RESULT_DISPLAY_TBALE, true); + + desc.setCreator(creator); + desc.setParamConfMap(confMap); + desc.setParamRunTimeMap(runtimeMap); + desc.setParamVarsMap(varMap); + desc.setLabelMap(labelMap); + desc.setSourceMap(sourceMap); + desc.setExecutionMap(executionMap); + desc.setSubmitUser(submitUsr); + desc.setProxyUser(proxyUsr); + + return desc; + } + + public static LinkisJobOper generateOperator(CliCtx ctx) { + LinkisJobOper linkisJobOperator = new LinkisJobOper(); + linkisJobOperator.setUJESClient(UJESClientFactory.getReusable(ctx.getVarAccess())); + linkisJobOperator.setServerUrl( + ctx.getVarAccess().getVar(String.class, CliKeys.LINKIS_COMMON_GATEWAY_URL)); + return linkisJobOperator; + } +} diff --git a/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/job/interactive/InteractiveJobResult.java b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/job/interactive/InteractiveJobResult.java new file mode 100644 index 0000000000..92999169ed --- /dev/null +++ b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/job/interactive/InteractiveJobResult.java @@ -0,0 +1,61 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.cli.application.interactor.job.interactive; + +import org.apache.linkis.cli.application.entity.job.JobResult; + +import java.util.Map; + +public class InteractiveJobResult implements JobResult { + private Boolean success; + private String message; + private Map extraMessage; + + public InteractiveJobResult(Boolean success, String message, Map extraMessage) { + this.success = success; + this.message = message; + this.extraMessage = extraMessage; + } + + @Override + public Boolean isSuccess() { + return success; + } + + @Override + public String getMessage() { + return message; + } + + public void setMessage(String message) { + this.message = message; + } + + @Override + public Map getExtraMessage() { + return extraMessage; + } + + public void setExtraMessage(Map extraMessage) { + this.extraMessage = extraMessage; + } + + public void setSuccess(Boolean success) { + this.success = success; + } +} diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/interactor/job/desc/LinkisJobManDesc.java b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/job/jobcmd/JobCmdDesc.java similarity index 82% rename from linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/interactor/job/desc/LinkisJobManDesc.java rename to linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/job/jobcmd/JobCmdDesc.java index da4f5d1c47..99f0365004 100644 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/interactor/job/desc/LinkisJobManDesc.java +++ b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/job/jobcmd/JobCmdDesc.java @@ -15,18 +15,25 @@ * limitations under the License. */ -package org.apache.linkis.cli.application.interactor.job.desc; - -import org.apache.linkis.cli.common.entity.job.JobDescription; +package org.apache.linkis.cli.application.interactor.job.jobcmd; import java.util.Map; -public class LinkisJobManDesc implements JobDescription { +public class JobCmdDesc { private String jobId; private String user; + private JobCmdSubType subType; private Map params; + public JobCmdSubType getSubType() { + return subType; + } + + public void setSubType(JobCmdSubType subType) { + this.subType = subType; + } + public String getJobID() { return jobId; } diff --git a/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/job/jobcmd/JobCmdDescBuilder.java b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/job/jobcmd/JobCmdDescBuilder.java new file mode 100644 index 0000000000..1e52e6e532 --- /dev/null +++ b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/job/jobcmd/JobCmdDescBuilder.java @@ -0,0 +1,67 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.cli.application.interactor.job.jobcmd; + +import org.apache.linkis.cli.application.constants.CliKeys; +import org.apache.linkis.cli.application.entity.context.CliCtx; +import org.apache.linkis.cli.application.utils.CliUtils; + +import org.apache.commons.lang3.StringUtils; + +import java.util.HashSet; +import java.util.Set; + +public class JobCmdDescBuilder { + public static JobCmdDesc build(CliCtx ctx) { + JobCmdDesc desc = new JobCmdDesc(); + String osUser = System.getProperty(CliKeys.LINUX_USER_KEY); + String[] adminUsers = StringUtils.split(CliKeys.ADMIN_USERS, ','); + Set adminSet = new HashSet<>(); + for (String admin : adminUsers) { + adminSet.add(admin); + } + String submitUsr = CliUtils.getSubmitUser(ctx.getVarAccess(), osUser, adminSet); + + JobCmdSubType subType = null; + + String jobId = null; + if (ctx.getVarAccess().hasVar(CliKeys.LINKIS_CLIENT_KILL_OPT)) { + jobId = ctx.getVarAccess().getVar(String.class, CliKeys.LINKIS_CLIENT_KILL_OPT); + subType = JobCmdSubType.KILL; + } else if (ctx.getVarAccess().hasVar(CliKeys.LINKIS_CLIENT_STATUS_OPT)) { + jobId = ctx.getVarAccess().getVar(String.class, CliKeys.LINKIS_CLIENT_STATUS_OPT); + subType = JobCmdSubType.STATUS; + } else if (ctx.getVarAccess().hasVar(CliKeys.LINKIS_CLIENT_DESC_OPT)) { + jobId = ctx.getVarAccess().getVar(String.class, CliKeys.LINKIS_CLIENT_DESC_OPT); + subType = JobCmdSubType.DESC; + } else if (ctx.getVarAccess().hasVar(CliKeys.LINKIS_CLIENT_LOG_OPT)) { + jobId = ctx.getVarAccess().getVar(String.class, CliKeys.LINKIS_CLIENT_LOG_OPT); + subType = JobCmdSubType.LOG; + } else if (ctx.getVarAccess().hasVar(CliKeys.LINKIS_CLIENT_RESULT_OPT)) { + jobId = ctx.getVarAccess().getVar(String.class, CliKeys.LINKIS_CLIENT_RESULT_OPT); + subType = JobCmdSubType.RESULT; + } else if (ctx.getVarAccess().hasVar(CliKeys.LINKIS_CLIENT_LIST_OPT)) { + jobId = ctx.getVarAccess().getVar(String.class, CliKeys.LINKIS_CLIENT_LIST_OPT); + subType = JobCmdSubType.LIST; + } + desc.setSubType(subType); + desc.setJobId(jobId); + desc.setUser(submitUsr); + return desc; + } +} diff --git a/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/job/jobcmd/JobCmdJob.java b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/job/jobcmd/JobCmdJob.java new file mode 100644 index 0000000000..e39be38f42 --- /dev/null +++ b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/job/jobcmd/JobCmdJob.java @@ -0,0 +1,212 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.cli.application.interactor.job.jobcmd; + +import org.apache.linkis.cli.application.constants.CliKeys; +import org.apache.linkis.cli.application.entity.context.CliCtx; +import org.apache.linkis.cli.application.entity.job.Job; +import org.apache.linkis.cli.application.entity.job.JobResult; +import org.apache.linkis.cli.application.exception.LinkisClientExecutionException; +import org.apache.linkis.cli.application.exception.LinkisClientRuntimeException; +import org.apache.linkis.cli.application.exception.error.CommonErrMsg; +import org.apache.linkis.cli.application.exception.error.ErrorLevel; +import org.apache.linkis.cli.application.interactor.job.common.JobKiller; +import org.apache.linkis.cli.application.interactor.job.common.KillResult; +import org.apache.linkis.cli.application.interactor.job.common.LogRetriever; +import org.apache.linkis.cli.application.interactor.job.common.ResultRetriever; +import org.apache.linkis.cli.application.operator.OperManager; +import org.apache.linkis.cli.application.operator.ujes.LinkisJobOper; +import org.apache.linkis.cli.application.operator.ujes.LinkisOperResultAdapter; +import org.apache.linkis.cli.application.present.JobInfoPresenter; +import org.apache.linkis.cli.application.present.LogPresenter; +import org.apache.linkis.cli.application.present.ResultPresenter; +import org.apache.linkis.cli.application.present.model.LinkisJobInfoModel; +import org.apache.linkis.cli.application.utils.LoggerManager; + +import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.exception.ExceptionUtils; + +import java.util.HashMap; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class JobCmdJob implements Job { + private static final Logger logger = LoggerFactory.getLogger(JobCmdJob.class); + + protected CliCtx ctx; + + protected LinkisJobOper oper; + + protected JobCmdDesc desc; + + @Override + public void build(CliCtx ctx) { + this.ctx = ctx; + this.desc = JobCmdDescBuilder.build(ctx); + this.oper = (LinkisJobOper) OperManager.getNew(CliKeys.Linkis_OPER, ctx); + } + + @Override + public JobResult run() { + JobCmdSubType subType = desc.getSubType(); + if (!(subType instanceof JobCmdSubType)) { + throw new LinkisClientExecutionException( + "EXE0030", + ErrorLevel.ERROR, + CommonErrMsg.ExecutionErr, + "JobSubType is not instance of JobManSubType"); + } + JobCmdJobResult result = new JobCmdJobResult(true, "Execute Success!!!", new HashMap<>()); + + switch (subType) { + case STATUS: + try { + LinkisOperResultAdapter jobInfoResult = + oper.queryJobInfo(desc.getUser(), desc.getJobID()); + LinkisJobInfoModel model = new LinkisJobInfoModel(); + model.buildModel(jobInfoResult); + new JobInfoPresenter().present(model); + } catch (Exception e) { + result.setSuccess(false); + result.setMessage(ExceptionUtils.getStackTrace(e)); + } + if (!result.isSuccess()) { + LoggerManager.getPlaintTextLogger() + .error("Failed to get job-info. Message: " + result.getMessage()); + } + return result; + case LOG: + try { + // get log while running + LinkisOperResultAdapter jobInfoResult = + oper.queryJobInfo(desc.getUser(), desc.getJobID()); + LogRetriever logRetriever = + new LogRetriever( + jobInfoResult.getUser(), + jobInfoResult.getJobID(), + jobInfoResult.getStrongerExecId(), + false, + oper, + new LogPresenter()); + // async because we need to query job status + logRetriever.retrieveLogAsync(); + logRetriever.waitIncLogComplete(); + } catch (Exception e) { + result.setSuccess(false); + result.setMessage(ExceptionUtils.getStackTrace(e)); + } + if (!result.isSuccess()) { + LoggerManager.getInformationLogger() + .error("Failed to get log. Message: " + result.getMessage()); + } + return result; + case RESULT: + // get log while running + LinkisOperResultAdapter jobInfoResult = oper.queryJobInfo(desc.getUser(), desc.getJobID()); + // get result-set + String outputPath = + ctx.getVarAccess().getVar(String.class, CliKeys.LINKIS_CLIENT_COMMON_OUTPUT_PATH); + ResultPresenter presenter; + if (StringUtils.isBlank(outputPath)) { + presenter = new ResultPresenter(); + } else { + presenter = new ResultPresenter(true, outputPath); + } + + ResultRetriever resultRetriever = + new ResultRetriever( + jobInfoResult.getUser(), + jobInfoResult.getJobID(), + jobInfoResult.getStrongerExecId(), + oper, + presenter); + + result = getResult(resultRetriever); + if (!result.isSuccess()) { + LoggerManager.getInformationLogger() + .error("Failed to get result. Message: " + result.getMessage()); + } + return result; + case KILL: + JobKiller jobKiller = new JobKiller(oper); + KillResult killResult; + try { + killResult = jobKiller.doKill(desc.getUser(), desc.getJobID()); + } catch (Exception e) { + killResult = + new KillResult( + false, + "Failed to kill job. Messgae: " + ExceptionUtils.getStackTrace(e), + new HashMap<>()); + } + if (killResult.isSuccess()) { + LoggerManager.getPlaintTextLogger().info("Kill Success. Current job-info:"); + } else { + LoggerManager.getPlaintTextLogger() + .error("Kill Failed. Messgae: " + killResult.getMessage() + "\n Current job-info:"); + } + try { + LinkisOperResultAdapter jobInfoResult2 = + oper.queryJobInfo(desc.getUser(), desc.getJobID()); + LinkisJobInfoModel model = new LinkisJobInfoModel(); + model.buildModel(jobInfoResult2); + new JobInfoPresenter().present(model); + } catch (Exception e) { + LoggerManager.getInformationLogger().error("Failed to get jobInfo", e); + } + return new JobCmdJobResult( + killResult.isSuccess(), killResult.getMessage(), killResult.getExtraMessage()); + // case LIST: + // break; + // case JOB_DESC: + // break; + default: + return new JobCmdJobResult( + false, "JobSubType + \"" + subType + "\" is not supported", new HashMap<>()); + } + } + + private JobCmdJobResult getResult(ResultRetriever resultRetriever) + throws LinkisClientRuntimeException { + JobCmdJobResult result = new JobCmdJobResult(true, "Execute Success!!!", new HashMap<>()); + try { + resultRetriever.retrieveResultSync(); + result.setSuccess(true); + result.setMessage("execute success!!!"); + } catch (LinkisClientExecutionException e) { + if (e.getCode().equals("EXE0037")) { + result.setSuccess(true); + result.setMessage("execute success!!!"); + LoggerManager.getInformationLogger().warn(e.getMessage()); + } else { + result.setSuccess(false); + result.setMessage("execute failed!!!\n" + ExceptionUtils.getStackTrace(e)); + } + resultRetriever.setResultFin(); // inform listener to stop + } catch (Exception e) { + result.setSuccess(false); + result.setMessage("execute failed!!!\n" + ExceptionUtils.getStackTrace(e)); + resultRetriever.setResultFin(); // inform listener to stop + } + return result; + } + + @Override + public void onDestroy() {} +} diff --git a/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/job/jobcmd/JobCmdJobResult.java b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/job/jobcmd/JobCmdJobResult.java new file mode 100644 index 0000000000..c5b9b43535 --- /dev/null +++ b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/job/jobcmd/JobCmdJobResult.java @@ -0,0 +1,61 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.cli.application.interactor.job.jobcmd; + +import org.apache.linkis.cli.application.entity.job.JobResult; + +import java.util.Map; + +public class JobCmdJobResult implements JobResult { + private Boolean success; + private String message; + private Map extraMessage; + + public JobCmdJobResult(Boolean success, String message, Map extraMessage) { + this.success = success; + this.message = message; + this.extraMessage = extraMessage; + } + + @Override + public Boolean isSuccess() { + return success; + } + + @Override + public String getMessage() { + return message; + } + + public void setMessage(String message) { + this.message = message; + } + + @Override + public Map getExtraMessage() { + return extraMessage; + } + + public void setExtraMessage(Map extraMessage) { + this.extraMessage = extraMessage; + } + + public void setSuccess(Boolean success) { + this.success = success; + } +} diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/interactor/job/subtype/LinkisManSubType.java b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/job/jobcmd/JobCmdSubType.java similarity index 81% rename from linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/interactor/job/subtype/LinkisManSubType.java rename to linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/job/jobcmd/JobCmdSubType.java index 5df4b07976..b2d298995a 100644 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/interactor/job/subtype/LinkisManSubType.java +++ b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/job/jobcmd/JobCmdSubType.java @@ -15,11 +15,9 @@ * limitations under the License. */ -package org.apache.linkis.cli.application.interactor.job.subtype; +package org.apache.linkis.cli.application.interactor.job.jobcmd; -import org.apache.linkis.cli.common.entity.job.JobSubType; - -public enum LinkisManSubType implements JobSubType { +public enum JobCmdSubType { KILL("kill"), LOG("log"), DESC("desc"), @@ -29,11 +27,10 @@ public enum LinkisManSubType implements JobSubType { private String name; - LinkisManSubType(String name) { + JobCmdSubType(String name) { this.name = name; } - @Override public String getName() { return this.name; } diff --git a/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/job/once/LinkisOnceJob.java b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/job/once/LinkisOnceJob.java new file mode 100644 index 0000000000..fac387998e --- /dev/null +++ b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/job/once/LinkisOnceJob.java @@ -0,0 +1,106 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.cli.application.interactor.job.once; + +import org.apache.linkis.cli.application.constants.CliKeys; +import org.apache.linkis.cli.application.entity.context.CliCtx; +import org.apache.linkis.cli.application.entity.job.Job; +import org.apache.linkis.cli.application.entity.job.JobResult; +import org.apache.linkis.cli.application.entity.job.JobStatus; +import org.apache.linkis.cli.application.operator.OperManager; +import org.apache.linkis.cli.application.operator.once.OnceJobOper; +import org.apache.linkis.cli.application.utils.CliUtils; +import org.apache.linkis.cli.application.utils.LoggerManager; +import org.apache.linkis.cli.application.utils.SchedulerManager; + +import java.util.HashMap; +import java.util.concurrent.CountDownLatch; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class LinkisOnceJob implements Job { + + private static final Logger logger = LoggerFactory.getLogger(LinkisOnceJob.class); + + private Boolean isAsync = false; + private OnceJobOper oper; + + @Override + public void build(CliCtx ctx) { + this.isAsync = + ctx.getVarAccess().getVarOrDefault(Boolean.class, CliKeys.LINKIS_CLIENT_ASYNC_OPT, false); + oper = (OnceJobOper) OperManager.getNew(CliKeys.LINKIS_ONCE, ctx); + } + + @Override + public JobResult run() { + StringBuilder infoBuilder = new StringBuilder(); + infoBuilder.append("connecting to linkis gateway:").append(oper.getServerUrl()); + LoggerManager.getInformationLogger().info(infoBuilder.toString()); + + /** submit */ + oper.submit(); + JobStatus jobStatus = oper.getStatus(); + infoBuilder.setLength(0); + infoBuilder.append("JobId:").append(oper.getJobID()).append(System.lineSeparator()); + LoggerManager.getPlaintTextLogger().info(infoBuilder.toString()); + if (isAsync && jobStatus != null && jobStatus.isJobSubmitted()) { + return new OnceJobResult(true, "Submit Success!!!", new HashMap<>()); + } + + /** getLog */ + CountDownLatch latch = new CountDownLatch(1); + try { + Thread logConsumer = new Thread(() -> ProcessLog(latch), "Log-Consumer"); + SchedulerManager.getCachedThreadPoolExecutor().execute(logConsumer); + } catch (Exception e) { + logger.warn("Failed to retrieve log", e); + } + + /** wait complete */ + oper.waitForComplete(); + try { + latch.await(); + } catch (Exception e) { + // ignore + } + + JobStatus finalStatus = oper.getStatus(); + + if (finalStatus.isJobSuccess()) { + return new OnceJobResult(true, "Execute Success!!!", new HashMap<>()); + } else { + return new OnceJobResult(false, "Execute Failure!!!", new HashMap<>()); + } + } + + @Override + public void onDestroy() { + oper.kill(); + } + + private void ProcessLog(CountDownLatch latch) { + while (!oper.isLogFin()) { + String log = oper.getCurrentLog(); + LoggerManager.getPlaintTextLogger().info(log); + CliUtils.doSleepQuietly(2000l); + } + latch.countDown(); + } +} diff --git a/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/job/once/OnceJobResult.java b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/job/once/OnceJobResult.java new file mode 100644 index 0000000000..2a14f76a6f --- /dev/null +++ b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/job/once/OnceJobResult.java @@ -0,0 +1,61 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.cli.application.interactor.job.once; + +import org.apache.linkis.cli.application.entity.job.JobResult; + +import java.util.Map; + +public class OnceJobResult implements JobResult { + private Boolean success; + private String message; + private Map extraMessage; + + public OnceJobResult(Boolean success, String message, Map extraMessage) { + this.success = success; + this.message = message; + this.extraMessage = extraMessage; + } + + @Override + public Boolean isSuccess() { + return success; + } + + @Override + public String getMessage() { + return message; + } + + public void setMessage(String message) { + this.message = message; + } + + @Override + public Map getExtraMessage() { + return extraMessage; + } + + public void setExtraMessage(Map extraMessage) { + this.extraMessage = extraMessage; + } + + public void setSuccess(Boolean success) { + this.success = success; + } +} diff --git a/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/job/version/VersionJob.java b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/job/version/VersionJob.java new file mode 100644 index 0000000000..599f97904f --- /dev/null +++ b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/job/version/VersionJob.java @@ -0,0 +1,48 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.cli.application.interactor.job.version; + +import org.apache.linkis.cli.application.constants.CliKeys; +import org.apache.linkis.cli.application.entity.context.CliCtx; +import org.apache.linkis.cli.application.entity.job.Job; +import org.apache.linkis.cli.application.entity.job.JobResult; +import org.apache.linkis.cli.application.utils.LoggerManager; + +import java.util.HashMap; +import java.util.Map; + +public class VersionJob implements Job { + private CliCtx ctx; + + @Override + public void build(CliCtx cliCtx) { + this.ctx = cliCtx; + } + + @Override + public JobResult run() { + String version = (String) ctx.getExtraMap().get(CliKeys.VERSION); + Map extraMap = new HashMap<>(); + extraMap.put(CliKeys.VERSION, version); + LoggerManager.getPlaintTextLogger().info("Version=" + version); + return new VersionJobResult(true, "ok", extraMap); + } + + @Override + public void onDestroy() {} +} diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/operator/JobOperatorBuilder.java b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/job/version/VersionJobResult.java similarity index 55% rename from linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/operator/JobOperatorBuilder.java rename to linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/job/version/VersionJobResult.java index 8727ec495e..e2f12cd7c2 100644 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/operator/JobOperatorBuilder.java +++ b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/job/version/VersionJobResult.java @@ -15,20 +15,35 @@ * limitations under the License. */ -package org.apache.linkis.cli.core.operator; +package org.apache.linkis.cli.application.interactor.job.version; -import org.apache.linkis.cli.common.entity.operator.JobOperator; -import org.apache.linkis.cli.common.entity.var.VarAccess; -import org.apache.linkis.cli.core.builder.BuildableByVarAccess; +import org.apache.linkis.cli.application.entity.job.JobResult; + +import java.util.Map; + +public class VersionJobResult implements JobResult { + private Boolean success; + private String message; + private Map extraMsg; + + public VersionJobResult(Boolean success, String message, Map extraMsg) { + this.success = success; + this.message = message; + this.extraMsg = extraMsg; + } + + @Override + public Boolean isSuccess() { + return success; + } -public abstract class JobOperatorBuilder extends BuildableByVarAccess { @Override - public JobOperatorBuilder setStdVarAccess(VarAccess varAccess) { - return (JobOperatorBuilder) super.setStdVarAccess(varAccess); + public String getMessage() { + return message; } @Override - public JobOperatorBuilder setSysVarAccess(VarAccess varAccess) { - return (JobOperatorBuilder) super.setSysVarAccess(varAccess); + public Map getExtraMessage() { + return extraMsg; } } diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-common/src/main/java/org/apache/linkis/cli/common/entity/properties/ClientProperties.java b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/properties/ClientProperties.java similarity index 95% rename from linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-common/src/main/java/org/apache/linkis/cli/common/entity/properties/ClientProperties.java rename to linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/properties/ClientProperties.java index 00c36a3032..6d165eeed6 100644 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-common/src/main/java/org/apache/linkis/cli/common/entity/properties/ClientProperties.java +++ b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/properties/ClientProperties.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.linkis.cli.common.entity.properties; +package org.apache.linkis.cli.application.interactor.properties; import java.util.HashMap; diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/properties/StdPropsLoader.java b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/properties/PropertiesLoader.java similarity index 82% rename from linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/properties/StdPropsLoader.java rename to linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/properties/PropertiesLoader.java index f6917206ab..d66cfe605d 100644 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/properties/StdPropsLoader.java +++ b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/properties/PropertiesLoader.java @@ -15,24 +15,22 @@ * limitations under the License. */ -package org.apache.linkis.cli.core.interactor.properties; +package org.apache.linkis.cli.application.interactor.properties; -import org.apache.linkis.cli.common.entity.properties.ClientProperties; -import org.apache.linkis.cli.common.exception.error.ErrorLevel; -import org.apache.linkis.cli.core.exception.PropsException; -import org.apache.linkis.cli.core.exception.error.CommonErrMsg; -import org.apache.linkis.cli.core.interactor.properties.reader.PropertiesReader; +import org.apache.linkis.cli.application.exception.PropsException; +import org.apache.linkis.cli.application.exception.error.CommonErrMsg; +import org.apache.linkis.cli.application.exception.error.ErrorLevel; +import org.apache.linkis.cli.application.interactor.properties.reader.PropertiesReader; import java.util.*; -public class StdPropsLoader implements PropertiesLoader { +public class PropertiesLoader { Map readersMap; - public StdPropsLoader() { + public PropertiesLoader() { this.readersMap = new HashMap<>(); } - @Override public PropertiesLoader setPropertiesReaders(PropertiesReader[] readers) { this.readersMap = new HashMap<>(); for (PropertiesReader reader : readers) { @@ -41,7 +39,6 @@ public PropertiesLoader setPropertiesReaders(PropertiesReader[] readers) { return this; } - @Override public PropertiesLoader addPropertiesReader(PropertiesReader reader) { if (reader != null) { readersMap.put(reader.getPropsId(), reader); @@ -49,7 +46,6 @@ public PropertiesLoader addPropertiesReader(PropertiesReader reader) { return this; } - @Override public PropertiesLoader addPropertiesReaders(PropertiesReader[] readers) { if (readers != null && readers.length > 0) { for (PropertiesReader reader : readers) { @@ -59,12 +55,10 @@ public PropertiesLoader addPropertiesReaders(PropertiesReader[] readers) { return this; } - @Override public void removePropertiesReader(String identifier) { readersMap.remove(identifier); } - @Override public ClientProperties[] loadProperties() { checkInit(); List propsList = new ArrayList<>(); @@ -81,7 +75,6 @@ public ClientProperties[] loadProperties() { return propsList.toArray(new ClientProperties[propsList.size()]); } - @Override public void checkInit() { if (readersMap == null || readersMap.size() == 0) { throw new PropsException( diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/properties/PropsFilesScanner.java b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/properties/PropsFilesScanner.java similarity index 81% rename from linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/properties/PropsFilesScanner.java rename to linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/properties/PropsFilesScanner.java index 2bd120e5ca..8e343d0948 100644 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/properties/PropsFilesScanner.java +++ b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/properties/PropsFilesScanner.java @@ -15,14 +15,14 @@ * limitations under the License. */ -package org.apache.linkis.cli.core.interactor.properties; +package org.apache.linkis.cli.application.interactor.properties; -import org.apache.linkis.cli.common.exception.error.ErrorLevel; -import org.apache.linkis.cli.core.constants.CommonConstants; -import org.apache.linkis.cli.core.exception.PropsException; -import org.apache.linkis.cli.core.exception.error.CommonErrMsg; -import org.apache.linkis.cli.core.interactor.properties.reader.PropertiesReader; -import org.apache.linkis.cli.core.interactor.properties.reader.PropsFileReader; +import org.apache.linkis.cli.application.constants.CliConstants; +import org.apache.linkis.cli.application.exception.PropsException; +import org.apache.linkis.cli.application.exception.error.CommonErrMsg; +import org.apache.linkis.cli.application.exception.error.ErrorLevel; +import org.apache.linkis.cli.application.interactor.properties.reader.PropertiesReader; +import org.apache.linkis.cli.application.interactor.properties.reader.PropsFileReader; import org.apache.commons.io.FileUtils; import org.apache.commons.lang3.StringUtils; @@ -51,7 +51,7 @@ public List getPropsFiles(String rootPath) { try { files = (List) - FileUtils.listFiles(new File(rootPath), CommonConstants.CONFIG_EXTENSION, false); + FileUtils.listFiles(new File(rootPath), CliConstants.CONFIG_EXTENSION, false); } catch (Exception e) { throw new PropsException( "PRP0005", diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/properties/reader/PropertiesReader.java b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/properties/reader/PropertiesReader.java similarity index 93% rename from linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/properties/reader/PropertiesReader.java rename to linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/properties/reader/PropertiesReader.java index 3c86f1b1c3..65bc2d1e06 100644 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/properties/reader/PropertiesReader.java +++ b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/properties/reader/PropertiesReader.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.linkis.cli.core.interactor.properties.reader; +package org.apache.linkis.cli.application.interactor.properties.reader; import java.util.Properties; diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/properties/reader/PropsFileReader.java b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/properties/reader/PropsFileReader.java similarity index 90% rename from linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/properties/reader/PropsFileReader.java rename to linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/properties/reader/PropsFileReader.java index 024a83311e..7bd23da140 100644 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/properties/reader/PropsFileReader.java +++ b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/properties/reader/PropsFileReader.java @@ -15,11 +15,11 @@ * limitations under the License. */ -package org.apache.linkis.cli.core.interactor.properties.reader; +package org.apache.linkis.cli.application.interactor.properties.reader; -import org.apache.linkis.cli.common.exception.error.ErrorLevel; -import org.apache.linkis.cli.core.exception.PropsException; -import org.apache.linkis.cli.core.exception.error.CommonErrMsg; +import org.apache.linkis.cli.application.exception.PropsException; +import org.apache.linkis.cli.application.exception.error.CommonErrMsg; +import org.apache.linkis.cli.application.exception.error.ErrorLevel; import org.apache.commons.lang3.StringUtils; diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/interactor/validate/LinkisSubmitValidator.java b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/validate/InteractiveDescValidator.java similarity index 76% rename from linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/interactor/validate/LinkisSubmitValidator.java rename to linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/validate/InteractiveDescValidator.java index fa0f31dc1e..03eef75612 100644 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/interactor/validate/LinkisSubmitValidator.java +++ b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/validate/InteractiveDescValidator.java @@ -18,67 +18,56 @@ package org.apache.linkis.cli.application.interactor.validate; import org.apache.linkis.cli.application.constants.LinkisKeys; -import org.apache.linkis.cli.application.interactor.job.LinkisSubmitJob; -import org.apache.linkis.cli.application.interactor.job.desc.LinkisSubmitDesc; -import org.apache.linkis.cli.common.entity.validate.Validator; -import org.apache.linkis.cli.common.exception.LinkisClientRuntimeException; -import org.apache.linkis.cli.common.exception.error.ErrorLevel; -import org.apache.linkis.cli.core.exception.ValidateException; -import org.apache.linkis.cli.core.exception.error.CommonErrMsg; +import org.apache.linkis.cli.application.exception.LinkisClientRuntimeException; +import org.apache.linkis.cli.application.exception.ValidateException; +import org.apache.linkis.cli.application.exception.error.CommonErrMsg; +import org.apache.linkis.cli.application.exception.error.ErrorLevel; +import org.apache.linkis.cli.application.interactor.job.interactive.InteractiveJobDesc; import org.apache.commons.lang3.StringUtils; import java.util.Map; -public class LinkisSubmitValidator implements Validator { - @Override - public void doValidation(Object input) throws LinkisClientRuntimeException { - if (!(input instanceof LinkisSubmitJob)) { - throw new ValidateException( - "VLD0007", - ErrorLevel.ERROR, - CommonErrMsg.ValidationErr, - "Input of LinkisSubmitValidator is not instance of LinkisSubmitJob. Type: " - + input.getClass().getCanonicalName()); - } +public class InteractiveDescValidator { + + public void doValidation(InteractiveJobDesc desc) throws LinkisClientRuntimeException { boolean ok = true; StringBuilder reasonSb = new StringBuilder(); - LinkisSubmitDesc submitDesc = ((LinkisSubmitJob) input).getJobDesc(); - if (StringUtils.isBlank(submitDesc.getSubmitUser())) { + if (StringUtils.isBlank(desc.getSubmitUser())) { reasonSb.append("Submit User cannot be empty or blank").append(System.lineSeparator()); ok = false; } - if (StringUtils.isBlank(submitDesc.getProxyUser())) { + if (StringUtils.isBlank(desc.getProxyUser())) { reasonSb .append("proxy(execute) User cannot be empty or blank") .append(System.lineSeparator()); ok = false; } - if (submitDesc.getLabelMap() == null) { + if (desc.getLabelMap() == null) { reasonSb.append("labelMap cannot be null").append(System.lineSeparator()); ok = false; } - if (submitDesc.getExecutionMap() == null) { + if (desc.getExecutionMap() == null) { reasonSb.append("ExecutionMap cannot be null").append(System.lineSeparator()); ok = false; } - if (submitDesc.getSourceMap() == null) { + if (desc.getSourceMap() == null) { reasonSb.append("SourceMap cannot be null").append(System.lineSeparator()); ok = false; } - if (submitDesc.getParamConfMap() == null) { + if (desc.getParamConfMap() == null) { reasonSb.append("startupMap cannot be null").append(System.lineSeparator()); ok = false; } - if (submitDesc.getParamVarsMap() == null) { + if (desc.getParamVarsMap() == null) { reasonSb.append("variableMap cannot be null").append(System.lineSeparator()); ok = false; } - if (submitDesc.getParamRunTimeMap() == null) { + if (desc.getParamRunTimeMap() == null) { reasonSb.append("runTimeMap cannot be null").append(System.lineSeparator()); ok = false; } - for (Map.Entry entry : submitDesc.getExecutionMap().entrySet()) { + for (Map.Entry entry : desc.getExecutionMap().entrySet()) { if (StringUtils.contains(entry.getKey(), " ")) { reasonSb .append("ExecutionMap key cannot contains space character. key: ") @@ -87,7 +76,7 @@ public void doValidation(Object input) throws LinkisClientRuntimeException { ok = false; } } - for (Map.Entry entry : submitDesc.getLabelMap().entrySet()) { + for (Map.Entry entry : desc.getLabelMap().entrySet()) { if (StringUtils.contains(entry.getKey(), " ")) { reasonSb .append("LabelMap key cannot contains space character. key: ") @@ -108,7 +97,7 @@ public void doValidation(Object input) throws LinkisClientRuntimeException { } } } - for (Map.Entry entry : submitDesc.getParamConfMap().entrySet()) { + for (Map.Entry entry : desc.getParamConfMap().entrySet()) { if (StringUtils.contains(entry.getKey(), " ")) { reasonSb .append("startUpMap key cannot contains space character. key: ") @@ -145,7 +134,7 @@ public void doValidation(Object input) throws LinkisClientRuntimeException { // } // } // } - for (Map.Entry entry : submitDesc.getParamVarsMap().entrySet()) { + for (Map.Entry entry : desc.getParamVarsMap().entrySet()) { if (StringUtils.contains(entry.getKey(), " ")) { reasonSb .append("variablesMap key cannot contains space character. key: ") @@ -164,7 +153,7 @@ public void doValidation(Object input) throws LinkisClientRuntimeException { // } // } } - for (Map.Entry entry : submitDesc.getSourceMap().entrySet()) { + for (Map.Entry entry : desc.getSourceMap().entrySet()) { if (StringUtils.contains(entry.getKey(), " ")) { reasonSb .append("sourceMap key cannot contains space character. key: ") @@ -185,30 +174,30 @@ public void doValidation(Object input) throws LinkisClientRuntimeException { } } } - if (StringUtils.isBlank((String) submitDesc.getLabelMap().get(LinkisKeys.KEY_ENGINETYPE))) { + if (StringUtils.isBlank((String) desc.getLabelMap().get(LinkisKeys.KEY_ENGINETYPE))) { reasonSb .append(LinkisKeys.KEY_ENGINETYPE) .append(" cannot be empty or blank") .append(System.lineSeparator()); ok = false; } - if (StringUtils.isBlank((String) submitDesc.getLabelMap().get(LinkisKeys.KEY_CODETYPE))) { + if (StringUtils.isBlank((String) desc.getLabelMap().get(LinkisKeys.KEY_CODETYPE))) { reasonSb .append(LinkisKeys.KEY_CODETYPE) .append(" cannot be empty or blank") .append(System.lineSeparator()); ok = false; } - if (StringUtils.isBlank((String) submitDesc.getSourceMap().get(LinkisKeys.KEY_SCRIPT_PATH))) { + if (StringUtils.isBlank((String) desc.getSourceMap().get(LinkisKeys.KEY_SCRIPT_PATH))) { reasonSb .append(LinkisKeys.KEY_SCRIPT_PATH) .append(" cannot be empty or blank") .append(System.lineSeparator()); ok = false; } - if (StringUtils.isBlank((String) submitDesc.getExecutionMap().get(LinkisKeys.KEY_CODE)) + if (StringUtils.isBlank((String) desc.getExecutionMap().get(LinkisKeys.KEY_CODE)) && StringUtils.indexOfIgnoreCase( - (String) submitDesc.getLabelMap().get(LinkisKeys.KEY_ENGINETYPE), "sqoop") + (String) desc.getLabelMap().get(LinkisKeys.KEY_ENGINETYPE), "sqoop") == -1) { reasonSb .append(LinkisKeys.KEY_CODE) @@ -216,15 +205,15 @@ public void doValidation(Object input) throws LinkisClientRuntimeException { .append(System.lineSeparator()); ok = false; } - if (StringUtils.isBlank((String) submitDesc.getLabelMap().get(LinkisKeys.KEY_USER_CREATOR))) { + if (StringUtils.isBlank((String) desc.getLabelMap().get(LinkisKeys.KEY_USER_CREATOR))) { reasonSb .append(LinkisKeys.KEY_USER_CREATOR) .append(" cannot be empty or blank") .append(System.lineSeparator()); ok = false; } else { - String userCreator = (String) submitDesc.getLabelMap().get(LinkisKeys.KEY_USER_CREATOR); - if (StringUtils.indexOf(submitDesc.getProxyUser(), "-") != -1) { + String userCreator = (String) desc.getLabelMap().get(LinkisKeys.KEY_USER_CREATOR); + if (StringUtils.indexOf(desc.getProxyUser(), "-") != -1) { reasonSb .append("\'proxyUser\' should not contain special character \'-\'") .append(System.lineSeparator()); diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/interactor/validate/LinkisManageValidator.java b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/validate/JobCmdDescValidator.java similarity index 60% rename from linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/interactor/validate/LinkisManageValidator.java rename to linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/validate/JobCmdDescValidator.java index 519a2ee85c..95b95d5833 100644 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/interactor/validate/LinkisManageValidator.java +++ b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/validate/JobCmdDescValidator.java @@ -17,30 +17,18 @@ package org.apache.linkis.cli.application.interactor.validate; -import org.apache.linkis.cli.application.interactor.job.LinkisManageJob; -import org.apache.linkis.cli.application.interactor.job.desc.LinkisJobManDesc; -import org.apache.linkis.cli.common.entity.validate.Validator; -import org.apache.linkis.cli.common.exception.LinkisClientRuntimeException; -import org.apache.linkis.cli.common.exception.error.ErrorLevel; -import org.apache.linkis.cli.core.exception.ValidateException; -import org.apache.linkis.cli.core.exception.error.CommonErrMsg; +import org.apache.linkis.cli.application.exception.LinkisClientRuntimeException; +import org.apache.linkis.cli.application.exception.ValidateException; +import org.apache.linkis.cli.application.exception.error.CommonErrMsg; +import org.apache.linkis.cli.application.exception.error.ErrorLevel; +import org.apache.linkis.cli.application.interactor.job.jobcmd.JobCmdDesc; import org.apache.commons.lang3.StringUtils; -public class LinkisManageValidator implements Validator { - @Override - public void doValidation(Object input) throws LinkisClientRuntimeException { - if (!(input instanceof LinkisManageJob)) { - throw new ValidateException( - "VLD0007", - ErrorLevel.ERROR, - CommonErrMsg.ValidationErr, - "Input of LinkisSubmitValidator is not instance of LinkisManageJob. Type: " - + input.getClass().getCanonicalName()); - } +public class JobCmdDescValidator { + public void doValidation(JobCmdDesc desc) throws LinkisClientRuntimeException { boolean ok = true; StringBuilder reasonSb = new StringBuilder(); - LinkisJobManDesc desc = ((LinkisManageJob) input).getJobDesc(); if (StringUtils.isBlank(desc.getJobID())) { reasonSb.append("jobId cannot be empty or blank").append(System.lineSeparator()); ok = false; diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/interactor/validate/LinkisOnceSubmitValidator.java b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/validate/OnceDescValidator.java similarity index 77% rename from linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/interactor/validate/LinkisOnceSubmitValidator.java rename to linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/validate/OnceDescValidator.java index fba6644700..f62cafe054 100644 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/interactor/validate/LinkisOnceSubmitValidator.java +++ b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/validate/OnceDescValidator.java @@ -18,71 +18,59 @@ package org.apache.linkis.cli.application.interactor.validate; import org.apache.linkis.cli.application.constants.LinkisKeys; -import org.apache.linkis.cli.application.interactor.job.LinkisOnceJob; -import org.apache.linkis.cli.application.interactor.job.desc.LinkisOnceDesc; -import org.apache.linkis.cli.common.entity.validate.Validator; -import org.apache.linkis.cli.common.exception.LinkisClientRuntimeException; -import org.apache.linkis.cli.common.exception.error.ErrorLevel; -import org.apache.linkis.cli.core.exception.ValidateException; -import org.apache.linkis.cli.core.exception.error.CommonErrMsg; +import org.apache.linkis.cli.application.exception.LinkisClientRuntimeException; +import org.apache.linkis.cli.application.exception.ValidateException; +import org.apache.linkis.cli.application.exception.error.CommonErrMsg; +import org.apache.linkis.cli.application.exception.error.ErrorLevel; +import org.apache.linkis.cli.application.operator.once.OnceJobDesc; import org.apache.commons.lang3.StringUtils; import java.util.Map; -public class LinkisOnceSubmitValidator implements Validator { - @Override - public void doValidation(Object input) throws LinkisClientRuntimeException { - if (!(input instanceof LinkisOnceJob)) { - throw new ValidateException( - "VLD0007", - ErrorLevel.ERROR, - CommonErrMsg.ValidationErr, - "Input of LinkisSubmitValidator is not instance of LinkisSubmitJob. Type: " - + input.getClass().getCanonicalName()); - } +public class OnceDescValidator { + public void doValidation(OnceJobDesc desc) throws LinkisClientRuntimeException { boolean ok = true; StringBuilder reasonSb = new StringBuilder(); - LinkisOnceDesc submitDesc = (LinkisOnceDesc) ((LinkisOnceJob) input).getJobDesc(); - if (StringUtils.isBlank(submitDesc.getSubmitUser())) { + if (StringUtils.isBlank(desc.getSubmitUser())) { reasonSb.append("Submit User cannot be empty or blank").append(System.lineSeparator()); ok = false; } - if (StringUtils.isBlank(submitDesc.getProxyUser())) { + if (StringUtils.isBlank(desc.getProxyUser())) { reasonSb .append("proxy(execute) User cannot be empty or blank") .append(System.lineSeparator()); ok = false; } - if (submitDesc.getLabelMap() == null) { + if (desc.getLabelMap() == null) { reasonSb.append("labelMap cannot be null").append(System.lineSeparator()); ok = false; } - if (submitDesc.getExecutionMap() == null) { + if (desc.getExecutionMap() == null) { reasonSb.append("ExecutionMap cannot be null").append(System.lineSeparator()); ok = false; } - if (submitDesc.getSourceMap() == null) { + if (desc.getSourceMap() == null) { reasonSb.append("SourceMap cannot be null").append(System.lineSeparator()); ok = false; } - if (submitDesc.getParamConfMap() == null) { + if (desc.getParamConfMap() == null) { reasonSb.append("startupMap cannot be null").append(System.lineSeparator()); ok = false; } - if (submitDesc.getParamVarsMap() == null) { + if (desc.getParamVarsMap() == null) { reasonSb.append("variableMap cannot be null").append(System.lineSeparator()); ok = false; } - if (submitDesc.getParamRunTimeMap() == null) { + if (desc.getParamRunTimeMap() == null) { reasonSb.append("runTimeMap cannot be null").append(System.lineSeparator()); ok = false; } - if (submitDesc.getJobContentMap() == null) { + if (desc.getJobContentMap() == null) { reasonSb.append("jobContentMap cannot be null").append(System.lineSeparator()); ok = false; } - for (Map.Entry entry : submitDesc.getExecutionMap().entrySet()) { + for (Map.Entry entry : desc.getExecutionMap().entrySet()) { if (StringUtils.contains(entry.getKey(), " ")) { reasonSb .append("ExecutionMap key cannot contains space character. key: ") @@ -91,7 +79,7 @@ public void doValidation(Object input) throws LinkisClientRuntimeException { ok = false; } } - for (Map.Entry entry : submitDesc.getLabelMap().entrySet()) { + for (Map.Entry entry : desc.getLabelMap().entrySet()) { if (StringUtils.contains(entry.getKey(), " ")) { reasonSb .append("LabelMap key cannot contains space character. key: ") @@ -112,7 +100,7 @@ public void doValidation(Object input) throws LinkisClientRuntimeException { } } } - for (Map.Entry entry : submitDesc.getParamConfMap().entrySet()) { + for (Map.Entry entry : desc.getParamConfMap().entrySet()) { if (StringUtils.contains(entry.getKey(), " ")) { reasonSb .append("startUpMap key cannot contains space character. key: ") @@ -149,7 +137,7 @@ public void doValidation(Object input) throws LinkisClientRuntimeException { // } // } // } - for (Map.Entry entry : submitDesc.getParamVarsMap().entrySet()) { + for (Map.Entry entry : desc.getParamVarsMap().entrySet()) { if (StringUtils.contains(entry.getKey(), " ")) { reasonSb .append("variablesMap key cannot contains space character. key: ") @@ -168,7 +156,7 @@ public void doValidation(Object input) throws LinkisClientRuntimeException { // } // } } - for (Map.Entry entry : submitDesc.getSourceMap().entrySet()) { + for (Map.Entry entry : desc.getSourceMap().entrySet()) { if (StringUtils.contains(entry.getKey(), " ")) { reasonSb .append("sourceMap key cannot contains space character. key: ") @@ -189,7 +177,7 @@ public void doValidation(Object input) throws LinkisClientRuntimeException { } } } - for (Map.Entry entry : submitDesc.getJobContentMap().entrySet()) { + for (Map.Entry entry : desc.getJobContentMap().entrySet()) { if (StringUtils.contains(entry.getKey(), " ")) { reasonSb .append("jobContentMap key cannot contains space character. key: ") @@ -208,30 +196,30 @@ public void doValidation(Object input) throws LinkisClientRuntimeException { // } // } } - if (StringUtils.isBlank((String) submitDesc.getLabelMap().get(LinkisKeys.KEY_ENGINETYPE))) { + if (StringUtils.isBlank((String) desc.getLabelMap().get(LinkisKeys.KEY_ENGINETYPE))) { reasonSb .append(LinkisKeys.KEY_ENGINETYPE) .append(" cannot be empty or blank") .append(System.lineSeparator()); ok = false; } - if (StringUtils.isBlank((String) submitDesc.getLabelMap().get(LinkisKeys.KEY_CODETYPE))) { + if (StringUtils.isBlank((String) desc.getLabelMap().get(LinkisKeys.KEY_CODETYPE))) { reasonSb .append(LinkisKeys.KEY_CODETYPE) .append(" cannot be empty or blank") .append(System.lineSeparator()); ok = false; } - if (StringUtils.isBlank((String) submitDesc.getSourceMap().get(LinkisKeys.KEY_SCRIPT_PATH))) { + if (StringUtils.isBlank((String) desc.getSourceMap().get(LinkisKeys.KEY_SCRIPT_PATH))) { reasonSb .append(LinkisKeys.KEY_SCRIPT_PATH) .append(" cannot be empty or blank") .append(System.lineSeparator()); ok = false; } - if (StringUtils.isBlank((String) submitDesc.getExecutionMap().get(LinkisKeys.KEY_CODE)) + if (StringUtils.isBlank((String) desc.getExecutionMap().get(LinkisKeys.KEY_CODE)) && StringUtils.indexOfIgnoreCase( - (String) submitDesc.getLabelMap().get(LinkisKeys.KEY_ENGINETYPE), "sqoop") + (String) desc.getLabelMap().get(LinkisKeys.KEY_ENGINETYPE), "sqoop") == -1) { reasonSb .append(LinkisKeys.KEY_CODE) @@ -239,15 +227,15 @@ public void doValidation(Object input) throws LinkisClientRuntimeException { .append(System.lineSeparator()); ok = false; } - if (StringUtils.isBlank((String) submitDesc.getLabelMap().get(LinkisKeys.KEY_USER_CREATOR))) { + if (StringUtils.isBlank((String) desc.getLabelMap().get(LinkisKeys.KEY_USER_CREATOR))) { reasonSb .append(LinkisKeys.KEY_USER_CREATOR) .append(" cannot be empty or blank") .append(System.lineSeparator()); ok = false; } else { - String userCreator = (String) submitDesc.getLabelMap().get(LinkisKeys.KEY_USER_CREATOR); - if (StringUtils.indexOf(submitDesc.getProxyUser(), "-") != -1) { + String userCreator = (String) desc.getLabelMap().get(LinkisKeys.KEY_USER_CREATOR); + if (StringUtils.indexOf(desc.getProxyUser(), "-") != -1) { reasonSb .append("\'proxyUser\' should not contain special character \'-\'") .append(System.lineSeparator()); diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/validate/ParamValidator.java b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/validate/ParamValidator.java similarity index 75% rename from linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/validate/ParamValidator.java rename to linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/validate/ParamValidator.java index abbc695976..b54dd6b723 100644 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/validate/ParamValidator.java +++ b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/validate/ParamValidator.java @@ -15,14 +15,12 @@ * limitations under the License. */ -package org.apache.linkis.cli.core.interactor.validate; +package org.apache.linkis.cli.application.interactor.validate; -import org.apache.linkis.cli.common.entity.command.Params; -import org.apache.linkis.cli.common.entity.validate.Validator; -import org.apache.linkis.cli.common.exception.LinkisClientRuntimeException; +import org.apache.linkis.cli.application.entity.command.Params; +import org.apache.linkis.cli.application.exception.LinkisClientRuntimeException; -public class ParamValidator implements Validator { - @Override +public class ParamValidator { public void doValidation(Object input) throws LinkisClientRuntimeException { if (!(input instanceof Params)) { // TODO:throw diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/validate/ParsedTplValidator.java b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/validate/ParsedTplValidator.java similarity index 69% rename from linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/validate/ParsedTplValidator.java rename to linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/validate/ParsedTplValidator.java index e7f14e2df1..a7ea05c72b 100644 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/validate/ParsedTplValidator.java +++ b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/validate/ParsedTplValidator.java @@ -15,15 +15,14 @@ * limitations under the License. */ -package org.apache.linkis.cli.core.interactor.validate; +package org.apache.linkis.cli.application.interactor.validate; -import org.apache.linkis.cli.common.entity.command.CmdOption; -import org.apache.linkis.cli.common.entity.command.CmdTemplate; -import org.apache.linkis.cli.common.entity.validate.Validator; -import org.apache.linkis.cli.common.exception.error.ErrorLevel; -import org.apache.linkis.cli.core.exception.CommandException; -import org.apache.linkis.cli.core.exception.ValidateException; -import org.apache.linkis.cli.core.exception.error.CommonErrMsg; +import org.apache.linkis.cli.application.entity.command.CmdOption; +import org.apache.linkis.cli.application.entity.command.CmdTemplate; +import org.apache.linkis.cli.application.exception.CommandException; +import org.apache.linkis.cli.application.exception.ValidateException; +import org.apache.linkis.cli.application.exception.error.CommonErrMsg; +import org.apache.linkis.cli.application.exception.error.ErrorLevel; import java.text.MessageFormat; import java.util.List; @@ -35,20 +34,10 @@ * 1. Check if there is missing or unknown option. 2. Call checkParam method for command-specific * validation. */ -public class ParsedTplValidator implements Validator { +public class ParsedTplValidator { private static final Logger logger = LoggerFactory.getLogger(ParsedTplValidator.class); - @Override - public void doValidation(Object input) throws CommandException { - if (!(input instanceof CmdTemplate)) { - throw new ValidateException( - "VLD0006", - ErrorLevel.ERROR, - CommonErrMsg.ValidationErr, - "Input of ParsedTplValidator is not instance of CmdTemplate"); - } - - CmdTemplate parsedTemplateCopy = (CmdTemplate) input; + public void doValidation(CmdTemplate parsedTemplateCopy) throws CommandException { String msg = "start validating command \"{0}\", template \"{1}\""; logger.info( diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/interactor/validate/UJESContextValidator.java b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/validate/UJESContextValidator.java similarity index 75% rename from linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/interactor/validate/UJESContextValidator.java rename to linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/validate/UJESContextValidator.java index 8c0653cb4f..9e5a9a6457 100644 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/interactor/validate/UJESContextValidator.java +++ b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/validate/UJESContextValidator.java @@ -18,28 +18,18 @@ package org.apache.linkis.cli.application.interactor.validate; import org.apache.linkis.cli.application.constants.LinkisConstants; +import org.apache.linkis.cli.application.exception.LinkisClientRuntimeException; +import org.apache.linkis.cli.application.exception.ValidateException; +import org.apache.linkis.cli.application.exception.error.CommonErrMsg; +import org.apache.linkis.cli.application.exception.error.ErrorLevel; import org.apache.linkis.cli.application.operator.ujes.UJESClientContext; -import org.apache.linkis.cli.common.entity.validate.Validator; -import org.apache.linkis.cli.common.exception.LinkisClientRuntimeException; -import org.apache.linkis.cli.common.exception.error.ErrorLevel; -import org.apache.linkis.cli.core.exception.ValidateException; -import org.apache.linkis.cli.core.exception.error.CommonErrMsg; import org.apache.commons.lang3.StringUtils; -public class UJESContextValidator implements Validator { - @Override - public void doValidation(Object input) throws LinkisClientRuntimeException { - if (!(input instanceof UJESClientContext)) { - throw new ValidateException( - "VLD0009", - ErrorLevel.ERROR, - CommonErrMsg.ValidationErr, - "Input of UJESContextValidator is not instance of UjesClientDriverContext"); - } +public class UJESContextValidator { + public void doValidation(UJESClientContext context) throws LinkisClientRuntimeException { boolean ok = true; StringBuilder reasonSb = new StringBuilder(); - UJESClientContext context = (UJESClientContext) input; if (StringUtils.isBlank(context.getGatewayUrl())) { reasonSb.append("gatewayUrl cannot be empty or blank").append(System.lineSeparator()); ok = false; diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/var/StdVarAccess.java b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/var/VarAccessImpl.java similarity index 88% rename from linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/var/StdVarAccess.java rename to linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/var/VarAccessImpl.java index 267dae2325..9a871828a3 100644 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/interactor/var/StdVarAccess.java +++ b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/interactor/var/VarAccessImpl.java @@ -15,18 +15,18 @@ * limitations under the License. */ -package org.apache.linkis.cli.core.interactor.var; - -import org.apache.linkis.cli.common.entity.command.ParamItem; -import org.apache.linkis.cli.common.entity.command.Params; -import org.apache.linkis.cli.common.entity.properties.ClientProperties; -import org.apache.linkis.cli.common.entity.var.VarAccess; -import org.apache.linkis.cli.common.exception.error.ErrorLevel; -import org.apache.linkis.cli.core.exception.VarAccessException; -import org.apache.linkis.cli.core.exception.error.CommonErrMsg; -import org.apache.linkis.cli.core.interactor.command.SpecialMap; -import org.apache.linkis.cli.core.utils.converter.AbstractStringConverter; -import org.apache.linkis.cli.core.utils.converter.PredefinedStringConverters; +package org.apache.linkis.cli.application.interactor.var; + +import org.apache.linkis.cli.application.entity.command.ParamItem; +import org.apache.linkis.cli.application.entity.command.Params; +import org.apache.linkis.cli.application.entity.var.VarAccess; +import org.apache.linkis.cli.application.exception.VarAccessException; +import org.apache.linkis.cli.application.exception.error.CommonErrMsg; +import org.apache.linkis.cli.application.exception.error.ErrorLevel; +import org.apache.linkis.cli.application.interactor.command.SpecialMap; +import org.apache.linkis.cli.application.interactor.command.template.converter.AbstractStringConverter; +import org.apache.linkis.cli.application.interactor.command.template.converter.PredefinedStringConverters; +import org.apache.linkis.cli.application.interactor.properties.ClientProperties; import org.apache.commons.lang3.StringUtils; @@ -35,14 +35,14 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -public class StdVarAccess implements VarAccess { - private static Logger logger = LoggerFactory.getLogger(StdVarAccess.class); +public class VarAccessImpl implements VarAccess { + private static Logger logger = LoggerFactory.getLogger(VarAccessImpl.class); private Params cmdParams; private ClientProperties userConf; private ClientProperties defaultConf; private Map subMapCache; - public StdVarAccess setCmdParams(Params cmdParams) { + public VarAccessImpl setCmdParams(Params cmdParams) { this.cmdParams = cmdParams; return this; } @@ -51,7 +51,7 @@ public Params getSubParam(String identifier) { return this.cmdParams; } - public StdVarAccess setUserConf(ClientProperties userConf) { + public VarAccessImpl setUserConf(ClientProperties userConf) { this.userConf = userConf; return this; } @@ -60,7 +60,7 @@ public ClientProperties getUserConf(String identifier) { return this.userConf; } - public StdVarAccess setDefaultConf(ClientProperties defaultConf) { + public VarAccessImpl setDefaultConf(ClientProperties defaultConf) { this.defaultConf = defaultConf; return this; } @@ -69,7 +69,7 @@ public ClientProperties getDefaultConf(String identifier) { return this.defaultConf; } - public StdVarAccess init() { + public VarAccessImpl init() { this.subMapCache = new HashMap<>(); putSubMapCache(subMapCache, cmdParams); return this; @@ -104,7 +104,6 @@ private void putSubMapCache(Map subMapCache, Params param) { } } - @Override public void checkInit() { if (this.cmdParams == null || this.defaultConf == null || this.subMapCache == null) { throw new VarAccessException( diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/observer/event/TriggerEvent.java b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/observer/event/FetchResultEvent.java similarity index 93% rename from linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/observer/event/TriggerEvent.java rename to linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/observer/event/FetchResultEvent.java index dada521b59..d538a40dff 100644 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/observer/event/TriggerEvent.java +++ b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/observer/event/FetchResultEvent.java @@ -17,4 +17,4 @@ package org.apache.linkis.cli.application.observer.event; -public class TriggerEvent extends SingleObserverEvent {} +public class FetchResultEvent extends SingleObserverEvent {} diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/observer/event/LinkisClientEvent.java b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/observer/event/LinkisClientEvent.java similarity index 100% rename from linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/observer/event/LinkisClientEvent.java rename to linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/observer/event/LinkisClientEvent.java diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/observer/event/LogStartEvent.java b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/observer/event/LogStartEvent.java similarity index 100% rename from linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/observer/event/LogStartEvent.java rename to linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/observer/event/LogStartEvent.java diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/observer/event/SingleObserverEvent.java b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/observer/event/SingleObserverEvent.java similarity index 100% rename from linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/observer/event/SingleObserverEvent.java rename to linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/observer/event/SingleObserverEvent.java diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/observer/listener/LinkisClientListener.java b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/observer/listener/LinkisClientListener.java similarity index 100% rename from linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/observer/listener/LinkisClientListener.java rename to linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/observer/listener/LinkisClientListener.java diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-common/src/main/java/org/apache/linkis/cli/common/entity/validate/Validator.java b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/operator/JobOperBuilder.java similarity index 76% rename from linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-common/src/main/java/org/apache/linkis/cli/common/entity/validate/Validator.java rename to linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/operator/JobOperBuilder.java index c86e475248..0e1604b204 100644 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-common/src/main/java/org/apache/linkis/cli/common/entity/validate/Validator.java +++ b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/operator/JobOperBuilder.java @@ -15,10 +15,11 @@ * limitations under the License. */ -package org.apache.linkis.cli.common.entity.validate; +package org.apache.linkis.cli.application.operator; -import org.apache.linkis.cli.common.exception.LinkisClientRuntimeException; +import org.apache.linkis.cli.application.entity.context.CliCtx; +import org.apache.linkis.cli.application.entity.operator.JobOper; -public interface Validator { - void doValidation(Object input) throws LinkisClientRuntimeException; +public interface JobOperBuilder { + JobOper build(CliCtx ctx); } diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/test/java/org/apache/linkis/cli/application/operator/OperatorUtilsTest.java b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/operator/OperManager.java similarity index 56% rename from linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/test/java/org/apache/linkis/cli/application/operator/OperatorUtilsTest.java rename to linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/operator/OperManager.java index 53dd359148..a41bfd5f9f 100644 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/test/java/org/apache/linkis/cli/application/operator/OperatorUtilsTest.java +++ b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/operator/OperManager.java @@ -17,19 +17,29 @@ package org.apache.linkis.cli.application.operator; -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.DisplayName; -import org.junit.jupiter.api.Test; +import org.apache.linkis.cli.application.entity.context.CliCtx; +import org.apache.linkis.cli.application.entity.operator.JobOper; -public class OperatorUtilsTest { +import java.util.HashMap; +import java.util.Map; - @Test - @DisplayName("getNumOfLinesTest") - public void getNumOfLinesTest() { +public class OperManager { - String str = "name, \n" + "names"; - int numOfLines = OperatorUtils.getNumOfLines(str); + private static Map builderMap = new HashMap<>(); - Assertions.assertTrue(2 == numOfLines); + public static void register(String name, JobOperBuilder builder) { + builderMap.put(name, builder); + } + + public static void remove(String name) { + builderMap.remove(name); + } + + public static JobOper getNew(String name, CliCtx ctx) { + JobOperBuilder builder = builderMap.get(name); + if (builder == null) { + return null; + } + return builder.build(ctx); } } diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/operator/once/LinkisNodeStatus.java b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/operator/once/LinkisNodeStatus.java similarity index 100% rename from linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/operator/once/LinkisNodeStatus.java rename to linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/operator/once/LinkisNodeStatus.java diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/operator/once/OnceJobConstants.java b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/operator/once/OnceJobConstants.java similarity index 100% rename from linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/operator/once/OnceJobConstants.java rename to linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/operator/once/OnceJobConstants.java diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/interactor/job/desc/LinkisOnceDesc.java b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/operator/once/OnceJobDesc.java similarity index 82% rename from linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/interactor/job/desc/LinkisOnceDesc.java rename to linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/operator/once/OnceJobDesc.java index a98d99383e..2bb63c0e0e 100644 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/interactor/job/desc/LinkisOnceDesc.java +++ b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/operator/once/OnceJobDesc.java @@ -15,18 +15,14 @@ * limitations under the License. */ -package org.apache.linkis.cli.application.interactor.job.desc; +package org.apache.linkis.cli.application.operator.once; -import org.apache.linkis.cli.common.entity.job.JobDescription; -import org.apache.linkis.cli.common.entity.var.VarAccess; +import org.apache.linkis.cli.application.entity.var.VarAccess; import java.util.Map; -public class LinkisOnceDesc implements JobDescription { - - private VarAccess stdVarAccess; - private VarAccess sysVarAccess; - +public class OnceJobDesc { + private VarAccess varAccess; private String submitUser; private String proxyUser; private String creator; @@ -38,20 +34,12 @@ public class LinkisOnceDesc implements JobDescription { private Map sourceMap; private Map jobContentMap; - public VarAccess getStdVarAccess() { - return stdVarAccess; - } - - public void setStdVarAccess(VarAccess stdVarAccess) { - this.stdVarAccess = stdVarAccess; - } - - public VarAccess getSysVarAccess() { - return sysVarAccess; + public VarAccess getVarAccess() { + return varAccess; } - public void setSysVarAccess(VarAccess sysVarAccess) { - this.sysVarAccess = sysVarAccess; + public void setVarAccess(VarAccess varAccess) { + this.varAccess = varAccess; } public String getSubmitUser() { diff --git a/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/operator/once/OnceJobOper.java b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/operator/once/OnceJobOper.java new file mode 100644 index 0000000000..85e2c16637 --- /dev/null +++ b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/operator/once/OnceJobOper.java @@ -0,0 +1,166 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.cli.application.operator.once; + +import org.apache.linkis.cli.application.constants.CliConstants; +import org.apache.linkis.cli.application.constants.CliKeys; +import org.apache.linkis.cli.application.constants.LinkisKeys; +import org.apache.linkis.cli.application.entity.job.JobStatus; +import org.apache.linkis.cli.application.entity.operator.JobOper; +import org.apache.linkis.cli.application.entity.var.VarAccess; +import org.apache.linkis.cli.application.exception.LinkisClientExecutionException; +import org.apache.linkis.cli.application.exception.error.CommonErrMsg; +import org.apache.linkis.cli.application.exception.error.ErrorLevel; +import org.apache.linkis.cli.application.interactor.job.common.LinkisJobStatus; +import org.apache.linkis.cli.application.operator.ujes.UJESClientFactory; +import org.apache.linkis.computation.client.LinkisJobBuilder$; +import org.apache.linkis.computation.client.once.simple.SimpleOnceJob; +import org.apache.linkis.computation.client.once.simple.SimpleOnceJobBuilder; +import org.apache.linkis.computation.client.once.simple.SubmittableSimpleOnceJob; +import org.apache.linkis.computation.client.operator.impl.EngineConnLogOperator; +import org.apache.linkis.computation.client.operator.impl.EngineConnLogs; + +import org.apache.commons.lang3.StringUtils; + +public class OnceJobOper implements JobOper { + LinkisJobStatus jobStatus = LinkisJobStatus.UNSUBMITTED; + EngineConnLogOperator logOperator = null; + private SimpleOnceJob onceJob; + private String serverUrl; + private String engineTypeForECM; + private Boolean isLogFin = false; + + public void init(OnceJobDesc desc) { + + VarAccess varAccess = desc.getVarAccess(); + + serverUrl = varAccess.getVar(String.class, CliKeys.LINKIS_COMMON_GATEWAY_URL); + + LinkisJobBuilder$.MODULE$.setDefaultClientConfig( + UJESClientFactory.generateDWSClientConfig(varAccess)); + LinkisJobBuilder$.MODULE$.setDefaultUJESClient(UJESClientFactory.getReusable(varAccess)); + + String engineTypeRaw = (String) desc.getLabelMap().get(LinkisKeys.KEY_ENGINETYPE); + engineTypeForECM = engineTypeRaw; + + if (StringUtils.isNotBlank(engineTypeRaw)) { + engineTypeForECM = StringUtils.split(engineTypeRaw, "-")[0]; + } else { + engineTypeForECM = ""; + } // TODO: remove parsing and let server side parse engineType + + onceJob = + new SimpleOnceJobBuilder() + .setCreateService(CliConstants.LINKIS_CLI) + .addExecuteUser(desc.getProxyUser()) + .setStartupParams(desc.getParamConfMap()) + .setLabels(desc.getLabelMap()) + .setRuntimeParams(desc.getParamRunTimeMap()) + .setSource(desc.getSourceMap()) + .setVariableMap(desc.getParamVarsMap()) + .setJobContent(desc.getJobContentMap()) + .build(); + } + + public String getServerUrl() { + return serverUrl; + } + + public SimpleOnceJob getOnceJob() { + return onceJob; + } + + public void setOnceJob(SimpleOnceJob onceJob) { + this.onceJob = onceJob; + } + + private void panicIfNull(Object obj) { + if (obj == null) { + throw new LinkisClientExecutionException( + "EXE0040", + ErrorLevel.ERROR, + CommonErrMsg.ExecutionErr, + "Instance of " + obj.getClass().getCanonicalName() + " is null"); + } + } + + public void submit() { + panicIfNull(onceJob); + if (!(onceJob instanceof SubmittableSimpleOnceJob)) { + throw new LinkisClientExecutionException( + "EXE0041", + ErrorLevel.ERROR, + CommonErrMsg.ExecutionErr, + "onceJob is not properly initiated"); + } + ((SubmittableSimpleOnceJob) onceJob).submit(); + } + + public void kill() { + panicIfNull(onceJob); + onceJob.kill(); + } + + public String getJobID() { + return onceJob.getId(); + } + + public String getUser() { + return "TODO"; + } + + public JobStatus getStatus() { + panicIfNull(onceJob); + String status = onceJob.getStatus(); + return LinkisJobStatus.convertFromNodeStatusString(status); + } + + public void waitForComplete() { + panicIfNull(onceJob); + onceJob.waitForCompleted(); + } + + public String getCurrentLog() { + panicIfNull(onceJob); + if (logOperator == null) { + logOperator = + (EngineConnLogOperator) onceJob.getOperator(EngineConnLogOperator.OPERATOR_NAME()); + logOperator.setECMServiceInstance( + ((SubmittableSimpleOnceJob) onceJob).getECMServiceInstance()); + logOperator.setEngineConnType(engineTypeForECM); + // logOperator.setPageSize(OnceJobConstants.MAX_LOG_SIZE_ONCE); + logOperator.setIgnoreKeywords(OnceJobConstants.LOG_IGNORE_KEYWORDS); + } + EngineConnLogs logs = + (EngineConnLogs) logOperator.apply(); // for some reason we have to add type conversion, + // otherwise mvn testCompile fails + StringBuilder logBuilder = new StringBuilder(); + for (String log : logs.logs()) { + logBuilder.append(log).append(System.lineSeparator()); + } + if ((logs.logs() == null || logs.logs().size() <= 0) && jobStatus.isJobFinishedState()) { + isLogFin = true; + } + return logBuilder.toString(); + // System.out.println(logs.logs().size()); + } + + public Boolean isLogFin() { + return isLogFin; + } +} diff --git a/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/operator/once/OnceOperBuilder.java b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/operator/once/OnceOperBuilder.java new file mode 100644 index 0000000000..299ce3e26f --- /dev/null +++ b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/operator/once/OnceOperBuilder.java @@ -0,0 +1,169 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.cli.application.operator.once; + +import org.apache.linkis.cli.application.constants.CliConstants; +import org.apache.linkis.cli.application.constants.CliKeys; +import org.apache.linkis.cli.application.constants.LinkisKeys; +import org.apache.linkis.cli.application.entity.context.CliCtx; +import org.apache.linkis.cli.application.entity.operator.JobOper; +import org.apache.linkis.cli.application.interactor.job.common.KeyParser; +import org.apache.linkis.cli.application.operator.JobOperBuilder; +import org.apache.linkis.cli.application.utils.CliUtils; + +import org.apache.commons.lang3.StringUtils; + +import java.util.*; + +public class OnceOperBuilder implements JobOperBuilder { + @Override + public JobOper build(CliCtx ctx) { + + OnceJobDesc desc = new OnceJobDesc(); + + Map confMap = ctx.getVarAccess().getVar(Map.class, CliKeys.JOB_PARAM_CONF); + Map runtimeMap = + ctx.getVarAccess().getVar(Map.class, CliKeys.JOB_PARAM_RUNTIME); + Map varMap = ctx.getVarAccess().getVar(Map.class, CliKeys.JOB_PARAM_VAR); + Map labelMap = ctx.getVarAccess().getVar(Map.class, CliKeys.JOB_LABEL); + Map sourceMap = ctx.getVarAccess().getVar(Map.class, CliKeys.JOB_SOURCE); + Map executionMap = ctx.getVarAccess().getVar(Map.class, CliKeys.JOB_EXEC); + Map jobContentMap = ctx.getVarAccess().getVar(Map.class, CliKeys.JOB_CONTENT); + + confMap = confMap == null ? new HashMap<>() : confMap; + runtimeMap = runtimeMap == null ? new HashMap<>() : runtimeMap; + varMap = varMap == null ? new HashMap<>() : varMap; + labelMap = labelMap == null ? new HashMap<>() : labelMap; + sourceMap = sourceMap == null ? new HashMap<>() : sourceMap; + executionMap = executionMap == null ? new HashMap<>() : executionMap; + jobContentMap = jobContentMap == null ? new HashMap<>() : jobContentMap; + + confMap = KeyParser.removePrefixForKeysInMap(confMap); + runtimeMap = KeyParser.removePrefixForKeysInMap(runtimeMap); + labelMap = KeyParser.removePrefixForKeysInMap(labelMap); + sourceMap = KeyParser.removePrefixForKeysInMap(sourceMap); + executionMap = KeyParser.removePrefixForKeysInMap(executionMap); + jobContentMap = KeyParser.removePrefixForKeysInMap(jobContentMap); + + for (String key : ctx.getVarAccess().getAllVarKeys()) { + Object val = ctx.getVarAccess().getVar(Object.class, key); + if (!(val instanceof Map) && val != null) { + // note that we allow it to overwrite existing values in map + if (StringUtils.startsWithIgnoreCase(key, CliKeys.JOB_PARAM_CONF)) { + KeyParser.removePrefixAndPutValToMap(confMap, key, val, CliKeys.JOB_PARAM_CONF); + } else if (StringUtils.startsWithIgnoreCase(key, CliKeys.JOB_PARAM_VAR)) { + KeyParser.removePrefixAndPutValToMap(varMap, key, val, CliKeys.JOB_PARAM_VAR); + } else if (StringUtils.startsWithIgnoreCase(key, CliKeys.JOB_PARAM_RUNTIME)) { + KeyParser.removePrefixAndPutValToMap(runtimeMap, key, val, CliKeys.JOB_PARAM_RUNTIME); + } else if (StringUtils.startsWithIgnoreCase(key, CliKeys.JOB_EXEC)) { + KeyParser.removePrefixAndPutValToMap(executionMap, key, val, CliKeys.JOB_EXEC); + } else if (StringUtils.startsWithIgnoreCase(key, CliKeys.JOB_LABEL)) { + KeyParser.removePrefixAndPutValToMap(labelMap, key, val, CliKeys.JOB_LABEL); + } else if (StringUtils.startsWithIgnoreCase(key, CliKeys.JOB_SOURCE)) { + KeyParser.removePrefixAndPutValToMap(sourceMap, key, val, CliKeys.JOB_SOURCE); + } else if (StringUtils.startsWithIgnoreCase(key, CliKeys.JOB_CONTENT)) { + KeyParser.removePrefixAndPutValToMap(jobContentMap, key, val, CliKeys.JOB_CONTENT); + } else if (StringUtils.startsWithIgnoreCase(key, CliKeys.LINKIS_CLIENT_COMMON)) { + // do nothing + } else { + // confMap.put(key, stdVarAccess.getVar(Object.class, key)); + } + } + } + + Boolean isAsync = + ctx.getVarAccess().getVarOrDefault(Boolean.class, CliKeys.LINKIS_CLIENT_ASYNC_OPT, false); + + String creator; + if (!isAsync) { + creator = + ctx.getVarAccess() + .getVarOrDefault( + String.class, CliKeys.JOB_COMMON_CREATOR, CliConstants.JOB_CREATOR_DEFAULT); + } else { + creator = + ctx.getVarAccess() + .getVarOrDefault( + String.class, CliKeys.JOB_COMMON_CREATOR, CliConstants.JOB_CREATOR_ASYNC_DEFAULT); + } + String code = ctx.getVarAccess().getVar(String.class, CliKeys.JOB_EXEC_CODE); + String engineType = ctx.getVarAccess().getVar(String.class, CliKeys.JOB_LABEL_ENGINE_TYPE); + String runType = ctx.getVarAccess().getVar(String.class, CliKeys.JOB_LABEL_CODE_TYPE); + String scriptPath = + ctx.getVarAccess() + .getVarOrDefault(String.class, CliKeys.JOB_SOURCE_SCRIPT_PATH, "LinkisCli"); + + String osUser = System.getProperty(CliKeys.LINUX_USER_KEY); + String[] adminUsers = StringUtils.split(CliKeys.ADMIN_USERS, ','); + Set adminSet = new HashSet<>(); + for (String admin : adminUsers) { + adminSet.add(admin); + } + String submitUsr = CliUtils.getSubmitUser(ctx.getVarAccess(), osUser, adminSet); + String proxyUsr = CliUtils.getProxyUser(ctx.getVarAccess(), submitUsr, adminSet); + + String enableExecuteOnce = + ctx.getVarAccess().getVarOrDefault(String.class, CliKeys.JOB_LABEL_EXECUTEONCE, "true"); + // default executeOnce-mode + if (Boolean.parseBoolean(enableExecuteOnce)) { + labelMap.put(LinkisKeys.KEY_EXECUTEONCE, ""); + } else { + labelMap.remove(LinkisKeys.KEY_EXECUTEONCE); + } + String codePath = ctx.getVarAccess().getVar(String.class, CliKeys.JOB_COMMON_CODE_PATH); + Object extraArgsObj = ctx.getVarAccess().getVar(Object.class, CliKeys.JOB_EXTRA_ARGUMENTS); + if (extraArgsObj != null + && extraArgsObj instanceof String[] + && StringUtils.isBlank(code) + && StringUtils.isBlank(codePath)) { + String[] extraArgs = (String[]) extraArgsObj; + codePath = extraArgs[0]; + if (extraArgs.length > 1) { + runtimeMap.put( + LinkisKeys.EXTRA_ARGUMENTS, Arrays.copyOfRange(extraArgs, 1, extraArgs.length)); + } + } + + if (StringUtils.isBlank(code) && StringUtils.isNotBlank(codePath)) { + code = CliUtils.readFile(codePath); + } + + executionMap.put(LinkisKeys.KEY_CODE, code); + labelMap.put(LinkisKeys.KEY_ENGINETYPE, engineType); + labelMap.put(LinkisKeys.KEY_CODETYPE, runType); + labelMap.put(LinkisKeys.KEY_USER_CREATOR, proxyUsr + "-" + creator); + sourceMap.put(LinkisKeys.KEY_SCRIPT_PATH, scriptPath); + runtimeMap.put(LinkisKeys.KEY_HIVE_RESULT_DISPLAY_TBALE, true); + + desc.setCreator(creator); + desc.setParamConfMap(confMap); + desc.setParamRunTimeMap(runtimeMap); + desc.setParamVarsMap(varMap); + desc.setLabelMap(labelMap); + desc.setSourceMap(sourceMap); + desc.setExecutionMap(executionMap); + desc.setSubmitUser(submitUsr); + desc.setProxyUser(proxyUsr); + desc.setJobContentMap(jobContentMap); + + OnceJobOper onceJobOper = new OnceJobOper(); + onceJobOper.init(desc); + + return onceJobOper; + } +} diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/operator/ujes/LinkisJobOperator.java b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/operator/ujes/LinkisJobOper.java similarity index 75% rename from linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/operator/ujes/LinkisJobOperator.java rename to linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/operator/ujes/LinkisJobOper.java index 87c49cd141..4c040c6f1e 100644 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/operator/ujes/LinkisJobOperator.java +++ b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/operator/ujes/LinkisJobOper.java @@ -18,15 +18,15 @@ package org.apache.linkis.cli.application.operator.ujes; import org.apache.linkis.cli.application.constants.LinkisKeys; -import org.apache.linkis.cli.application.interactor.job.desc.LinkisSubmitDesc; +import org.apache.linkis.cli.application.entity.operator.JobOper; +import org.apache.linkis.cli.application.exception.LinkisClientExecutionException; +import org.apache.linkis.cli.application.exception.LinkisClientRuntimeException; +import org.apache.linkis.cli.application.exception.error.CommonErrMsg; +import org.apache.linkis.cli.application.exception.error.ErrorLevel; +import org.apache.linkis.cli.application.interactor.job.interactive.InteractiveJobDesc; import org.apache.linkis.cli.application.operator.ujes.result.OpenLogResult2; import org.apache.linkis.cli.application.operator.ujes.result.ResultSetResult2; -import org.apache.linkis.cli.application.utils.Utils; -import org.apache.linkis.cli.common.entity.operator.JobOperator; -import org.apache.linkis.cli.common.exception.LinkisClientRuntimeException; -import org.apache.linkis.cli.common.exception.error.ErrorLevel; -import org.apache.linkis.cli.core.exception.LinkisClientExecutionException; -import org.apache.linkis.cli.core.exception.error.CommonErrMsg; +import org.apache.linkis.cli.application.utils.CliUtils; import org.apache.linkis.common.exception.LinkisException; import org.apache.linkis.ujes.client.UJESClient; import org.apache.linkis.ujes.client.request.JobSubmitAction; @@ -43,9 +43,9 @@ import org.slf4j.LoggerFactory; /** Based on UjesClient */ -public class LinkisJobOperator implements JobOperator { +public class LinkisJobOper implements JobOper { protected UJESClient client; - private Logger logger = LoggerFactory.getLogger(LinkisJobOperator.class); + private Logger logger = LoggerFactory.getLogger(LinkisJobOper.class); private String serverUrl; public UJESClient getUJESClient() { @@ -87,7 +87,7 @@ public void checkInit() throws LinkisClientRuntimeException { * @param * @return */ - public LinkisOperResultAdapter submit(LinkisSubmitDesc jobDesc) + public LinkisOperResultAdapter submit(InteractiveJobDesc jobDesc) throws LinkisClientRuntimeException { checkInit(); JobSubmitResult jobSubmitResult; @@ -104,7 +104,7 @@ public LinkisOperResultAdapter submit(LinkisSubmitDesc jobDesc) .setLabels(jobDesc.getLabelMap()) .setSource(jobDesc.getSourceMap()) .build(); - logger.info("Request info to Linkis: \n{}", Utils.GSON.toJson(jobSubmitAction)); + logger.info("Request info to Linkis: \n{}", CliUtils.GSON.toJson(jobSubmitAction)); /* Old API */ // JobExecuteAction jobExecuteAction = JobExecuteAction.builder() @@ -127,7 +127,7 @@ public LinkisOperResultAdapter submit(LinkisSubmitDesc jobDesc) // jobExecuteResult = client.execute(jobExecuteAction); jobSubmitResult = client.submit(jobSubmitAction); - logger.info("Response info from Linkis: \n{}", Utils.GSON.toJson(jobSubmitAction)); + logger.info("Response info from Linkis: \n{}", CliUtils.GSON.toJson(jobSubmitAction)); } catch (Exception e) { // must throw if exception @@ -142,7 +142,8 @@ public LinkisOperResultAdapter submit(LinkisSubmitDesc jobDesc) if (jobSubmitResult == null) { reason = "JobSubmitResult is null"; } else if (0 != jobSubmitResult.getStatus()) { - reason = "server returns non-zero status-code"; + reason = "server returns non-zero status-code. "; + reason += jobSubmitResult.getMessage(); } else { reason = "server returns blank TaskId"; } @@ -191,13 +192,14 @@ public LinkisOperResultAdapter queryJobStatus(String user, String taskID, String while (retryTime++ < MAX_RETRY_TIME) { try { jobStatusResult = client.status(executeResult); - logger.debug("job-status: " + Utils.GSON.toJson(jobStatusResult)); + logger.debug("job-status: " + CliUtils.GSON.toJson(jobStatusResult)); if (jobStatusResult == null || 0 != jobStatusResult.getStatus()) { String reason; if (jobStatusResult == null) { reason = "jobStatusResult is null"; } else { - reason = "server returns non-zero status-code"; + reason = "server returns non-zero status-code. "; + reason += jobStatusResult.getMessage(); } String msg = MessageFormat.format( @@ -224,14 +226,15 @@ public LinkisOperResultAdapter queryJobStatus(String user, String taskID, String "EXE0013", ErrorLevel.ERROR, CommonErrMsg.ExecutionErr, msg, e); } } - Utils.doSleepQuietly(UJESConstants.DRIVER_QUERY_SLEEP_MILLS); + CliUtils.doSleepQuietly(UJESConstants.DRIVER_QUERY_SLEEP_MILLS); } if (jobStatusResult == null || 0 != jobStatusResult.getStatus()) { String reason; if (jobStatusResult == null) { reason = "jobStatusResult is null"; } else { - reason = "server returns non-zero status-code"; + reason = "server returns non-zero status-code. "; + reason += jobStatusResult.getMessage(); } String msg = MessageFormat.format( @@ -255,17 +258,18 @@ private JobInfoResult queryJobInfoInternal(String user, String taskID) while (retryTime++ < MAX_RETRY_TIME) { try { jobInfoResult = client.getJobInfo(executeResult); - logger.debug("job-info: " + Utils.GSON.toJson(jobInfoResult)); + logger.debug("job-info: " + CliUtils.GSON.toJson(jobInfoResult)); if (jobInfoResult == null || 0 != jobInfoResult.getStatus()) { String reason; if (jobInfoResult == null) { reason = "JobInfoResult is null"; } else { - reason = "server returns non-zero status-code"; + reason = "server returns non-zero status-code. "; + reason += jobInfoResult.getMessage(); } String msg = MessageFormat.format( - "Get job info failed. retry time : {0}/{1}. taskID={0}, Reason: {1}", + "Get job info failed. retry time : {0}/{1}. taskID={2}, Reason: {3}", retryTime, MAX_RETRY_TIME, taskID, reason); logger.debug( @@ -288,14 +292,15 @@ private JobInfoResult queryJobInfoInternal(String user, String taskID) "EXE0013", ErrorLevel.ERROR, CommonErrMsg.ExecutionErr, msg, e); } } - Utils.doSleepQuietly(UJESConstants.DRIVER_QUERY_SLEEP_MILLS); + CliUtils.doSleepQuietly(UJESConstants.DRIVER_QUERY_SLEEP_MILLS); } if (jobInfoResult == null || 0 != jobInfoResult.getStatus()) { String reason; if (jobInfoResult == null) { reason = "JobInfoResult is null"; } else { - reason = "server returns non-zero status-code"; + reason = "server returns non-zero status-code. "; + reason += jobInfoResult.getMessage(); } String msg = MessageFormat.format( @@ -313,61 +318,30 @@ public LinkisOperResultAdapter queryRunTimeLogFromLine( jobExecuteResult.setUser(user); jobExecuteResult.setTaskID(taskID); jobExecuteResult.setExecID(execID); - - JobLogResult logResult = null; - int retryTime = 0; - final int MAX_RETRY_TIME = UJESConstants.DRIVER_REQUEST_MAX_RETRY_TIME; - - while (retryTime++ < MAX_RETRY_TIME) { - try { - logResult = client.log(jobExecuteResult, fromLine, UJESConstants.MAX_LOG_SIZE); - logger.debug("runtime-log-result:" + Utils.GSON.toJson(logResult)); - if (logResult == null || 0 != logResult.getStatus()) { - String reason; - if (logResult == null) { - reason = "JobLogResult is null"; - } else { - reason = "server returns non-zero status-code"; - } - String msg = - MessageFormat.format( - "Get log failed. retry time : {0}/{1}. taskID={2}. Reason: {3}", - retryTime, MAX_RETRY_TIME, taskID, reason); - logger.debug( - "", - new LinkisClientExecutionException( - "EXE0015", ErrorLevel.ERROR, CommonErrMsg.ExecutionErr, msg)); + try { + JobLogResult logResult = client.log(jobExecuteResult, fromLine, UJESConstants.MAX_LOG_SIZE); + logger.debug("runtime-log-result:" + CliUtils.GSON.toJson(logResult)); + if (logResult == null || 0 != logResult.getStatus()) { + String reason; + if (logResult == null) { + reason = "JobLogResult is null"; } else { - break; + reason = "server returns non-zero status-code. "; + reason += logResult.getMessage(); } - } catch (Exception e) { String msg = - MessageFormat.format("Get log failed. Retry time : {0}/{1}", retryTime, MAX_RETRY_TIME); - // logger.warn("", new LinkisClientExecutionException("EXE0016", - // ErrorLevel.ERROR, CommonErrMsg.ExecutionErr, msg)); - if (retryTime >= MAX_RETRY_TIME) { - throw new LinkisClientExecutionException( - "EXE0016", ErrorLevel.ERROR, CommonErrMsg.ExecutionErr, msg, e); - } - } - Utils.doSleepQuietly(UJESConstants.DRIVER_QUERY_SLEEP_MILLS); - } - if (logResult == null || 0 != logResult.getStatus()) { - String reason; - if (logResult == null) { - reason = "JobLogResult is null"; - } else { - reason = "server returns non-zero status-code"; + MessageFormat.format("Get log failed. taskID={0}. Reason: {1}", taskID, reason); + Exception e = + new LinkisClientExecutionException( + "EXE0016", ErrorLevel.ERROR, CommonErrMsg.ExecutionErr, msg); + logger.debug("", e); + throw e; } - String msg = - MessageFormat.format( - "Get log failed. Retry exhausted. taskID={0}, Reason: {1}", taskID, reason); - // logger.warn("", new LinkisClientExecutionException("EXE0016", - // ErrorLevel.ERROR, CommonErrMsg.ExecutionErr, msg)); + return new UJESResultAdapter(logResult); + } catch (Exception e) { throw new LinkisClientExecutionException( - "EXE0016", ErrorLevel.ERROR, CommonErrMsg.ExecutionErr, msg); + "EXE0016", ErrorLevel.ERROR, CommonErrMsg.ExecutionErr, "Get log failed.", e); } - return new UJESResultAdapter(logResult); } public LinkisOperResultAdapter queryPersistedLogFromLine( @@ -379,80 +353,39 @@ public LinkisOperResultAdapter queryPersistedLogFromLine( private OpenLogResult queryPersistedLogInternal(String logPath, String user, String taskID) throws LinkisClientRuntimeException { checkInit(); - int retryCnt = 0; - final int MAX_RETRY_TIMES = UJESConstants.DRIVER_REQUEST_MAX_RETRY_TIME; - OpenLogResult openLogResult = null; - - while (retryCnt++ < MAX_RETRY_TIMES) { - try { - openLogResult = - client.openLog( - OpenLogAction.newBuilder().setLogPath(logPath).setProxyUser(user).build()); - logger.debug("persisted-log-result:" + Utils.GSON.toJson(openLogResult)); - if (openLogResult == null - || 0 != openLogResult.getStatus() - || StringUtils.isBlank(openLogResult.getLog()[UJESConstants.IDX_FOR_LOG_TYPE_ALL])) { - String reason; - if (openLogResult == null) { - reason = "OpenLogResult is null"; - } else if (0 != openLogResult.getStatus()) { - reason = "server returns non-zero status-code"; - } else { - reason = "server returns empty log"; - } - String msg = - MessageFormat.format( - "Get log from openLog failed. retry time : {0}/{1}. taskID={2}. Reason: {3}", - retryCnt, MAX_RETRY_TIMES, taskID, reason); - logger.debug(msg); + try { + OpenLogResult openLogResult = + client.openLog(OpenLogAction.newBuilder().setLogPath(logPath).setProxyUser(user).build()); + logger.debug("persisted-log-result:" + CliUtils.GSON.toJson(openLogResult)); + if (openLogResult == null + || 0 != openLogResult.getStatus() + || StringUtils.isBlank(openLogResult.getLog()[UJESConstants.IDX_FOR_LOG_TYPE_ALL])) { + String reason; + if (openLogResult == null) { + reason = "OpenLogResult is null"; + } else if (0 != openLogResult.getStatus()) { + reason = "server returns non-zero status-code. "; + reason += openLogResult.getMessage(); } else { - break; + reason = "server returns empty log"; } - } catch (Exception e) { String msg = MessageFormat.format( - "Get log from openLog failed. retry time : {0}/{1}", retryCnt, MAX_RETRY_TIMES); - if (e instanceof LinkisException) { - msg += " " + e.toString(); - } - logger.debug(msg, e); - if (retryCnt >= MAX_RETRY_TIMES) { - throw new LinkisClientExecutionException( - "EXE0017", - ErrorLevel.ERROR, - CommonErrMsg.ExecutionErr, - "Get log from openLog failed. Retry exhausted. taskID=" + taskID, - e); - } - } - Utils.doSleepQuietly(UJESConstants.DRIVER_QUERY_SLEEP_MILLS); - } - if (openLogResult == null - || 0 != openLogResult.getStatus() - || StringUtils.isBlank(openLogResult.getLog()[UJESConstants.IDX_FOR_LOG_TYPE_ALL])) { - String reason; - if (openLogResult == null) { - reason = "OpenLogResult is null"; - } else if (0 != openLogResult.getStatus()) { - reason = "server returns non-zero status-code"; - } else { - reason = "server returns empty log"; - } - String msg = - MessageFormat.format( - "Get log from openLog failed. retry time : {0}/{1}. taskID={2}. Reason: {3}", - retryCnt, MAX_RETRY_TIMES, taskID, reason); - logger.debug(msg); - if (retryCnt >= MAX_RETRY_TIMES) { - msg = - MessageFormat.format( - "Get log from openLog failed. Retry exhausted. taskID={0}, Reason: {1}", - taskID, reason); + "Get log from openLog failed. taskID={0}. Reason: {1}", taskID, reason); + logger.debug(msg); throw new LinkisClientExecutionException( "EXE0017", ErrorLevel.ERROR, CommonErrMsg.ExecutionErr, msg); } + return openLogResult; + } catch (Exception e) { + String msg = "Get log from openLog failed."; + if (e instanceof LinkisException) { + msg += " " + e.toString(); + } + logger.debug(msg, e); + throw new LinkisClientExecutionException( + "EXE0017", ErrorLevel.ERROR, CommonErrMsg.ExecutionErr, msg); } - return openLogResult; } public UJESResultAdapter queryProgress(String user, String taskID, String execId) @@ -475,7 +408,8 @@ public UJESResultAdapter queryProgress(String user, String taskID, String execId if (jobProgressResult == null) { reason = "JobProgressResult is null"; } else { - reason = "server returns non-zero status-code"; + reason = "server returns non-zero status-code. "; + reason += jobProgressResult.getMessage(); } String msg = MessageFormat.format( @@ -502,7 +436,7 @@ public UJESResultAdapter queryProgress(String user, String taskID, String execId e); } } - Utils.doSleepQuietly(UJESConstants.DRIVER_QUERY_SLEEP_MILLS); + CliUtils.doSleepQuietly(UJESConstants.DRIVER_QUERY_SLEEP_MILLS); } if (jobProgressResult == null || 0 != jobProgressResult.getStatus()) { @@ -510,7 +444,8 @@ public UJESResultAdapter queryProgress(String user, String taskID, String execId if (jobProgressResult == null) { reason = "JobProgressResult is null"; } else { - reason = "server returns non-zero status-code"; + reason = "server returns non-zero status-code. "; + reason += jobProgressResult.getMessage(); } String msg = MessageFormat.format( @@ -588,7 +523,7 @@ public LinkisOperResultAdapter queryResultSetPaths( e); } } - Utils.doSleepQuietly(UJESConstants.DRIVER_QUERY_SLEEP_MILLS); + CliUtils.doSleepQuietly(UJESConstants.DRIVER_QUERY_SLEEP_MILLS); } if (resultSetArray == null || 0 == resultSetArray.length) { String reason; @@ -623,13 +558,14 @@ public LinkisOperResultAdapter queryResultSetGivenResultSetPath( .setPageSize(pageSize) .build(); result = client.resultSet(action); - logger.debug("resultset-result:" + Utils.GSON.toJson(result)); + logger.debug("resultset-result:" + CliUtils.GSON.toJson(result)); if (result == null || 0 != result.getStatus()) { String reason; if (result == null) { reason = "array is null"; } else { - reason = "server returns non-zero status-code"; + reason = "server returns non-zero status-code. "; + reason += result.getMessage(); } String msg = MessageFormat.format( @@ -656,14 +592,15 @@ public LinkisOperResultAdapter queryResultSetGivenResultSetPath( e); } } - Utils.doSleepQuietly(UJESConstants.DRIVER_QUERY_SLEEP_MILLS); + CliUtils.doSleepQuietly(UJESConstants.DRIVER_QUERY_SLEEP_MILLS); } if (result == null || 0 != result.getStatus()) { String reason; if (result == null) { reason = "ResultSetResult is null"; } else { - reason = "server returns non-zero status-code"; + reason = "server returns non-zero status-code. "; + reason += result.getMessage(); } String msg = MessageFormat.format( @@ -691,13 +628,14 @@ public LinkisOperResultAdapter kill(String user, String taskId, String execId) killRequest.setTaskID(taskId); killRequest.setExecID(execId); result = client.kill(killRequest); - logger.debug("job-kill-result:" + Utils.GSON.toJson(result)); + logger.debug("job-kill-result:" + CliUtils.GSON.toJson(result)); if (result == null || 0 != result.getStatus()) { String reason; if (result == null) { reason = "result is null"; } else { - reason = "server returns non-zero status-code"; + reason = "server returns non-zero status-code. "; + reason += result.getMessage(); } String msg = MessageFormat.format( @@ -725,14 +663,15 @@ public LinkisOperResultAdapter kill(String user, String taskId, String execId) e); } } - Utils.doSleepQuietly(UJESConstants.DRIVER_QUERY_SLEEP_MILLS); + CliUtils.doSleepQuietly(UJESConstants.DRIVER_QUERY_SLEEP_MILLS); } if (result == null || 0 != result.getStatus()) { String reason; if (result == null) { reason = "result is null"; } else { - reason = "server returns non-zero status-code"; + reason = "server returns non-zero status-code. "; + reason += result.getMessage(); } String msg = MessageFormat.format( diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/operator/ujes/LinkisOperatorBuilder.java b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/operator/ujes/LinkisOperBuilder.java similarity index 60% rename from linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/operator/ujes/LinkisOperatorBuilder.java rename to linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/operator/ujes/LinkisOperBuilder.java index 466bb9b7c0..318688c565 100644 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/operator/ujes/LinkisOperatorBuilder.java +++ b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/operator/ujes/LinkisOperBuilder.java @@ -17,28 +17,23 @@ package org.apache.linkis.cli.application.operator.ujes; -import org.apache.linkis.cli.application.constants.AppKeys; -import org.apache.linkis.cli.core.operator.JobOperatorBuilder; +import org.apache.linkis.cli.application.constants.CliKeys; +import org.apache.linkis.cli.application.entity.context.CliCtx; +import org.apache.linkis.cli.application.entity.operator.JobOper; +import org.apache.linkis.cli.application.operator.JobOperBuilder; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -public class LinkisOperatorBuilder extends JobOperatorBuilder { - private static Logger logger = LoggerFactory.getLogger(LinkisOperatorBuilder.class); +public class LinkisOperBuilder implements JobOperBuilder { + private static Logger logger = LoggerFactory.getLogger(LinkisOperBuilder.class); @Override - public LinkisJobOperator build() { - - ((LinkisJobOperator) targetObj) - .setUJESClient(UJESClientFactory.getReusable(stdVarAccess, sysVarAccess)); - ((LinkisJobOperator) targetObj) - .setServerUrl(stdVarAccess.getVar(String.class, AppKeys.LINKIS_COMMON_GATEWAY_URL)); - - return (LinkisJobOperator) super.build(); - } - - @Override - protected LinkisJobOperator getTargetNewInstance() { - return new LinkisJobOperator(); + public JobOper build(CliCtx ctx) { + LinkisJobOper jobOper = new LinkisJobOper(); + jobOper.setUJESClient(UJESClientFactory.getReusable(ctx.getVarAccess())); + jobOper.setServerUrl( + ctx.getVarAccess().getVar(String.class, CliKeys.LINKIS_COMMON_GATEWAY_URL)); + return jobOper; } } diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/operator/ujes/LinkisOperResultAdapter.java b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/operator/ujes/LinkisOperResultAdapter.java similarity index 90% rename from linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/operator/ujes/LinkisOperResultAdapter.java rename to linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/operator/ujes/LinkisOperResultAdapter.java index 457a7ba56d..bd22444a2d 100644 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/operator/ujes/LinkisOperResultAdapter.java +++ b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/operator/ujes/LinkisOperResultAdapter.java @@ -17,8 +17,8 @@ package org.apache.linkis.cli.application.operator.ujes; -import org.apache.linkis.cli.application.interactor.job.data.LinkisResultSet; -import org.apache.linkis.cli.common.entity.job.JobStatus; +import org.apache.linkis.cli.application.entity.job.JobStatus; +import org.apache.linkis.cli.application.interactor.job.common.ResultSet; import java.util.Date; @@ -52,7 +52,7 @@ public interface LinkisOperResultAdapter { Boolean hasNextLogLine(); - LinkisResultSet getResultContent(); + ResultSet getResultContent(); Boolean resultHasNextPage(); diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/operator/ujes/UJESClientContext.java b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/operator/ujes/UJESClientContext.java similarity index 100% rename from linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/operator/ujes/UJESClientContext.java rename to linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/operator/ujes/UJESClientContext.java diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/operator/ujes/UJESClientFactory.java b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/operator/ujes/UJESClientFactory.java similarity index 79% rename from linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/operator/ujes/UJESClientFactory.java rename to linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/operator/ujes/UJESClientFactory.java index 2767929e8d..a082fce198 100644 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/operator/ujes/UJESClientFactory.java +++ b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/operator/ujes/UJESClientFactory.java @@ -17,16 +17,15 @@ package org.apache.linkis.cli.application.operator.ujes; -import org.apache.linkis.cli.application.constants.AppKeys; +import org.apache.linkis.cli.application.constants.CliKeys; import org.apache.linkis.cli.application.constants.LinkisConstants; +import org.apache.linkis.cli.application.entity.var.VarAccess; +import org.apache.linkis.cli.application.exception.BuilderException; +import org.apache.linkis.cli.application.exception.LinkisClientExecutionException; +import org.apache.linkis.cli.application.exception.error.CommonErrMsg; +import org.apache.linkis.cli.application.exception.error.ErrorLevel; import org.apache.linkis.cli.application.interactor.validate.UJESContextValidator; -import org.apache.linkis.cli.application.utils.Utils; -import org.apache.linkis.cli.common.entity.validate.Validator; -import org.apache.linkis.cli.common.entity.var.VarAccess; -import org.apache.linkis.cli.common.exception.error.ErrorLevel; -import org.apache.linkis.cli.core.exception.BuilderException; -import org.apache.linkis.cli.core.exception.LinkisClientExecutionException; -import org.apache.linkis.cli.core.exception.error.CommonErrMsg; +import org.apache.linkis.cli.application.utils.CliUtils; import org.apache.linkis.httpclient.authentication.AuthenticationStrategy; import org.apache.linkis.httpclient.dws.authentication.StaticAuthenticationStrategy; import org.apache.linkis.httpclient.dws.authentication.TokenAuthenticationStrategy; @@ -47,20 +46,20 @@ public class UJESClientFactory { private static UJESClient client; - public static UJESClient getReusable(VarAccess stdVarAccess, VarAccess sysVarAccess) { + public static UJESClient getReusable(VarAccess stdVarAccess) { if (client == null) { synchronized (UJESClientFactory.class) { if (client == null) { - client = getNew(stdVarAccess, sysVarAccess); + client = getNew(stdVarAccess); } } } return client; } - public static UJESClient getNew(VarAccess stdVarAccess, VarAccess sysVarAccess) { + public static UJESClient getNew(VarAccess stdVarAccess) { try { - DWSClientConfig config = generateDWSClientConfig(stdVarAccess, sysVarAccess); + DWSClientConfig config = generateDWSClientConfig(stdVarAccess); UJESClient ret = new UJESClientImpl(config); logger.info("Linkis ujes client inited."); return ret; @@ -70,9 +69,8 @@ public static UJESClient getNew(VarAccess stdVarAccess, VarAccess sysVarAccess) } } - public static DWSClientConfig generateDWSClientConfig( - VarAccess stdVarAccess, VarAccess sysVarAccess) { - UJESClientContext context = generateContext(stdVarAccess, sysVarAccess); + public static DWSClientConfig generateDWSClientConfig(VarAccess stdVarAccess) { + UJESClientContext context = generateContext(stdVarAccess); try { AuthenticationStrategy authenticationStrategy; if (StringUtils.isBlank(context.getAuthenticationStrategyStr()) @@ -117,7 +115,7 @@ public static DWSClientConfig generateDWSClientConfig( public static DWSClientConfig generateDWSClientConfigForBML( VarAccess stdVarAccess, VarAccess sysVarAccess) { - UJESClientContext context = generateContext(stdVarAccess, sysVarAccess); + UJESClientContext context = generateContext(stdVarAccess); try { AuthenticationStrategy authenticationStrategy; if (StringUtils.isBlank(context.getAuthenticationStrategyStr()) @@ -160,8 +158,8 @@ public static DWSClientConfig generateDWSClientConfigForBML( } } - private static UJESClientContext generateContext(VarAccess stdVarAccess, VarAccess sysVarAccess) { - String gatewayUrl = stdVarAccess.getVar(String.class, AppKeys.LINKIS_COMMON_GATEWAY_URL); + private static UJESClientContext generateContext(VarAccess stdVarAccess) { + String gatewayUrl = stdVarAccess.getVar(String.class, CliKeys.LINKIS_COMMON_GATEWAY_URL); if (StringUtils.isBlank(gatewayUrl)) { throw new BuilderException( "BLD0007", @@ -170,33 +168,33 @@ private static UJESClientContext generateContext(VarAccess stdVarAccess, VarAcce "Cannot build UjesClientDriverContext: gatewayUrl is empty"); } - String authKey = stdVarAccess.getVar(String.class, AppKeys.LINKIS_COMMON_TOKEN_KEY); - String authValue = stdVarAccess.getVar(String.class, AppKeys.LINKIS_COMMON_TOKEN_VALUE); + String authKey = stdVarAccess.getVar(String.class, CliKeys.LINKIS_COMMON_TOKEN_KEY); + String authValue = stdVarAccess.getVar(String.class, CliKeys.LINKIS_COMMON_TOKEN_VALUE); String authenticationStrategy = stdVarAccess.getVarOrDefault( String.class, - AppKeys.LINKIS_COMMON_AUTHENTICATION_STRATEGY, + CliKeys.LINKIS_COMMON_AUTHENTICATION_STRATEGY, LinkisConstants.AUTH_STRATEGY_STATIC); long connectionTimeout = stdVarAccess.getVarOrDefault( - Long.class, AppKeys.UJESCLIENT_COMMON_CONNECTT_TIMEOUT, 30000L); + Long.class, CliKeys.UJESCLIENT_COMMON_CONNECTT_TIMEOUT, 30000L); boolean discoveryEnabled = stdVarAccess.getVarOrDefault( - Boolean.class, AppKeys.UJESCLIENT_COMMON_DISCOVERY_ENABLED, false); + Boolean.class, CliKeys.UJESCLIENT_COMMON_DISCOVERY_ENABLED, false); boolean loadBalancerEnabled = stdVarAccess.getVarOrDefault( - Boolean.class, AppKeys.UJESCLIENT_COMMON_LOADBALANCER_ENABLED, true); + Boolean.class, CliKeys.UJESCLIENT_COMMON_LOADBALANCER_ENABLED, true); int maxConnectionSize = stdVarAccess.getVarOrDefault( - Integer.class, AppKeys.UJESCLIENT_COMMON_MAX_CONNECTION_SIZE, 5); + Integer.class, CliKeys.UJESCLIENT_COMMON_MAX_CONNECTION_SIZE, 5); boolean retryEnabled = - stdVarAccess.getVarOrDefault(Boolean.class, AppKeys.UJESCLIENT_COMMON_RETRY_ENABLED, false); + stdVarAccess.getVarOrDefault(Boolean.class, CliKeys.UJESCLIENT_COMMON_RETRY_ENABLED, false); long readTimeout = - stdVarAccess.getVarOrDefault(Long.class, AppKeys.UJESCLIENT_COMMON_READTIMEOUT, 30000L); + stdVarAccess.getVarOrDefault(Long.class, CliKeys.UJESCLIENT_COMMON_READTIMEOUT, 30000L); String dwsVersion = - stdVarAccess.getVarOrDefault(String.class, AppKeys.UJESCLIENT_COMMON_DWS_VERSION, "v1"); + stdVarAccess.getVarOrDefault(String.class, CliKeys.UJESCLIENT_COMMON_DWS_VERSION, "v1"); UJESClientContext context = new UJESClientContext(); @@ -212,8 +210,8 @@ private static UJESClientContext generateContext(VarAccess stdVarAccess, VarAcce context.setReadTimeoutMills(readTimeout); context.setDwsVersion(dwsVersion); - logger.info("==========UJES_CTX============\n" + Utils.GSON.toJson(context)); - Validator ctxValidator = new UJESContextValidator(); + logger.info("==========UJES_CTX============\n" + CliUtils.GSON.toJson(context)); + UJESContextValidator ctxValidator = new UJESContextValidator(); ctxValidator.doValidation(context); return context; } diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/operator/ujes/UJESConstants.java b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/operator/ujes/UJESConstants.java similarity index 100% rename from linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/operator/ujes/UJESConstants.java rename to linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/operator/ujes/UJESConstants.java diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/operator/ujes/UJESResultAdapter.java b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/operator/ujes/UJESResultAdapter.java similarity index 90% rename from linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/operator/ujes/UJESResultAdapter.java rename to linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/operator/ujes/UJESResultAdapter.java index d402e01fe8..6c746ff57c 100644 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/operator/ujes/UJESResultAdapter.java +++ b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/operator/ujes/UJESResultAdapter.java @@ -18,17 +18,16 @@ package org.apache.linkis.cli.application.operator.ujes; import org.apache.linkis.cli.application.constants.LinkisKeys; -import org.apache.linkis.cli.application.interactor.job.LinkisJobStatus; -import org.apache.linkis.cli.application.interactor.job.data.LinkisResultSet; -import org.apache.linkis.cli.application.operator.OperatorUtils; +import org.apache.linkis.cli.application.entity.job.JobStatus; +import org.apache.linkis.cli.application.exception.TransformerException; +import org.apache.linkis.cli.application.exception.error.CommonErrMsg; +import org.apache.linkis.cli.application.exception.error.ErrorLevel; +import org.apache.linkis.cli.application.interactor.job.common.LinkisJobStatus; +import org.apache.linkis.cli.application.interactor.job.common.ResultSet; import org.apache.linkis.cli.application.operator.ujes.result.OpenLogResult2; import org.apache.linkis.cli.application.operator.ujes.result.ResultSetResult2; import org.apache.linkis.cli.application.operator.ujes.result.UJESResult; -import org.apache.linkis.cli.application.utils.Utils; -import org.apache.linkis.cli.common.entity.job.JobStatus; -import org.apache.linkis.cli.common.exception.error.ErrorLevel; -import org.apache.linkis.cli.core.exception.TransformerException; -import org.apache.linkis.cli.core.exception.error.CommonErrMsg; +import org.apache.linkis.cli.application.utils.CliUtils; import org.apache.linkis.httpclient.dws.response.DWSResult; import org.apache.linkis.ujes.client.request.UserAction; import org.apache.linkis.ujes.client.response.JobInfoResult; @@ -170,7 +169,7 @@ public String getStrongerExecId() { execId = (String) ((JobInfoResult) result).getTask().get(LinkisKeys.KEY_STRONGER_EXECID); } } - if (Utils.isValidExecId(execId)) { + if (CliUtils.isValidExecId(execId)) { return execId; } return null; @@ -277,8 +276,7 @@ public String getLog() { ((OpenLogResult2) result).getResult().getLog()[UJESConstants.IDX_FOR_LOG_TYPE_ALL]; Integer fromLine = ((OpenLogResult2) result).getFromLine(); return StringUtils.substring( - allLog, - OperatorUtils.getFirstIndexSkippingLines(allLog, fromLine == null ? 0 : fromLine)); + allLog, getFirstIndexSkippingLines(allLog, fromLine == null ? 0 : fromLine)); } return null; } @@ -294,7 +292,7 @@ public Integer getNextLogLine() { if (result instanceof OpenLogResult2 && ((OpenLogResult2) result).getResult() != null && ((OpenLogResult2) result).getResult().getLog() != null) { - return OperatorUtils.getNumOfLines( + return getNumOfLines( ((OpenLogResult2) result).getResult().getLog()[UJESConstants.IDX_FOR_LOG_TYPE_ALL]); } return null; @@ -315,7 +313,7 @@ public Boolean hasNextLogLine() { } @Override - public LinkisResultSet getResultContent() { + public ResultSet getResultContent() { if (result == null) { return null; } @@ -323,7 +321,7 @@ public LinkisResultSet getResultContent() { && ((ResultSetResult2) result).getResultSetResult() != null && ((ResultSetResult2) result).getResultSetResult().getFileContent() != null && ((ResultSetResult2) result).getResultSetResult().getMetadata() != null) { - LinkisResultSet ret = new LinkisResultSet(); + ResultSet ret = new ResultSet(); ret.setResultsetIdx(((ResultSetResult2) result).getIdxResultSet()); if (((ResultSetResult2) result).getResultSetResult().getMetadata() != null) { ret.setResultMeta( @@ -560,4 +558,54 @@ public String getRequestApplicationName() { } return null; } + + private int getNumOfLines(String str) { + if (str == null || str.length() == 0) { + return 0; + } + int lines = 1; + int len = str.length(); + for (int pos = 0; pos < len; pos++) { + char c = str.charAt(pos); + if (c == '\r') { + lines++; + if (pos + 1 < len && str.charAt(pos + 1) == '\n') { + pos++; + } + } else if (c == '\n') { + lines++; + } + } + return lines; + } + + private int getFirstIndexSkippingLines(String str, Integer lines) { + if (str == null || str.length() == 0 || lines < 0) { + return -1; + } + if (lines == 0) { + return 0; + } + + int curLineIdx = 0; + int len = str.length(); + for (int pos = 0; pos < len; pos++) { + char c = str.charAt(pos); + if (c == '\r') { + curLineIdx++; + if (pos + 1 < len && str.charAt(pos + 1) == '\n') { + pos++; + } + } else if (c == '\n') { + curLineIdx++; + } else { + continue; + } + + if (curLineIdx >= lines) { + return pos + 1; + } + } + return -1; + } } diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/operator/ujes/result/OpenLogResult2.java b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/operator/ujes/result/OpenLogResult2.java similarity index 100% rename from linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/operator/ujes/result/OpenLogResult2.java rename to linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/operator/ujes/result/OpenLogResult2.java diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/operator/ujes/result/ResultSetResult2.java b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/operator/ujes/result/ResultSetResult2.java similarity index 100% rename from linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/operator/ujes/result/ResultSetResult2.java rename to linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/operator/ujes/result/ResultSetResult2.java diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/operator/ujes/result/UJESResult.java b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/operator/ujes/result/UJESResult.java similarity index 100% rename from linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/operator/ujes/result/UJESResult.java rename to linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/operator/ujes/result/UJESResult.java diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/present/HelpInfoPresenter.java b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/present/HelpPresenter.java similarity index 65% rename from linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/present/HelpInfoPresenter.java rename to linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/present/HelpPresenter.java index 186081a560..f02be7fefc 100644 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/present/HelpInfoPresenter.java +++ b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/present/HelpPresenter.java @@ -15,23 +15,20 @@ * limitations under the License. */ -package org.apache.linkis.cli.core.present; +package org.apache.linkis.cli.application.present; -import org.apache.linkis.cli.common.entity.command.CmdOption; -import org.apache.linkis.cli.common.entity.command.CmdTemplate; -import org.apache.linkis.cli.common.entity.present.Model; -import org.apache.linkis.cli.common.entity.present.PresentWay; -import org.apache.linkis.cli.common.entity.present.Presenter; -import org.apache.linkis.cli.common.exception.error.ErrorLevel; -import org.apache.linkis.cli.core.exception.PresenterException; -import org.apache.linkis.cli.core.exception.error.CommonErrMsg; -import org.apache.linkis.cli.core.interactor.command.template.option.MapOption; -import org.apache.linkis.cli.core.interactor.command.template.option.Parameter; -import org.apache.linkis.cli.core.interactor.command.template.option.StdOption; -import org.apache.linkis.cli.core.present.display.DisplayOperator; -import org.apache.linkis.cli.core.present.display.StdOutWriter; -import org.apache.linkis.cli.core.present.display.data.StdoutDisplayData; -import org.apache.linkis.cli.core.present.model.HelpInfoModel; +import org.apache.linkis.cli.application.entity.command.CmdOption; +import org.apache.linkis.cli.application.entity.command.CmdTemplate; +import org.apache.linkis.cli.application.entity.present.Model; +import org.apache.linkis.cli.application.entity.present.Presenter; +import org.apache.linkis.cli.application.exception.PresenterException; +import org.apache.linkis.cli.application.exception.error.CommonErrMsg; +import org.apache.linkis.cli.application.exception.error.ErrorLevel; +import org.apache.linkis.cli.application.interactor.command.template.option.MapOption; +import org.apache.linkis.cli.application.interactor.command.template.option.Parameter; +import org.apache.linkis.cli.application.interactor.command.template.option.StdOption; +import org.apache.linkis.cli.application.present.model.HelpInfoModel; +import org.apache.linkis.cli.application.utils.LoggerManager; import java.util.ArrayList; import java.util.List; @@ -39,12 +36,11 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -public class HelpInfoPresenter implements Presenter { - private static Logger logger = LoggerFactory.getLogger(HelpInfoPresenter.class); - DisplayOperator driver = new StdOutWriter(); +public class HelpPresenter implements Presenter { + private static Logger logger = LoggerFactory.getLogger(HelpPresenter.class); @Override - public void present(Model model, PresentWay presentWay) { + public void present(Model model) { if (!(model instanceof HelpInfoModel)) { throw new PresenterException( "PST0010", @@ -52,15 +48,12 @@ public void present(Model model, PresentWay presentWay) { CommonErrMsg.PresenterErr, "Input for HelpInfoPresenter is not instance of model"); } - if (driver == null) { - throw new PresenterException( - "PST0007", ErrorLevel.ERROR, CommonErrMsg.PresenterErr, "Driver is null"); - } + HelpInfoModel helpInfoModel = (HelpInfoModel) model; String helpInfo = getUsage(helpInfoModel.getTemplate()); - driver.doOutput(new StdoutDisplayData(helpInfo)); + LoggerManager.getPlaintTextLogger().info(helpInfo); } /** Help info for sub-command */ diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/present/PresentModeImpl.java b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/present/JobInfoPresenter.java similarity index 63% rename from linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/present/PresentModeImpl.java rename to linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/present/JobInfoPresenter.java index aee032e3a4..3a264f827c 100644 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/present/PresentModeImpl.java +++ b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/present/JobInfoPresenter.java @@ -15,22 +15,16 @@ * limitations under the License. */ -package org.apache.linkis.cli.core.present; +package org.apache.linkis.cli.application.present; -public enum PresentModeImpl implements PresentMode { - STDOUT("stdout", 0), - TEXT_FILE("text_file", 1); - - private String name; - private int id; - - PresentModeImpl(String name, int id) { - this.name = name; - this.id = id; - } +import org.apache.linkis.cli.application.entity.present.Model; +import org.apache.linkis.cli.application.entity.present.Presenter; +import org.apache.linkis.cli.application.utils.CliUtils; +import org.apache.linkis.cli.application.utils.LoggerManager; +public class JobInfoPresenter implements Presenter { @Override - public String getName() { - return this.name(); + public void present(Model model) { + LoggerManager.getPlaintTextLogger().info(CliUtils.GSON.toJson(model)); } } diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/present/LinkisLogPresenter.java b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/present/LogPresenter.java similarity index 61% rename from linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/present/LinkisLogPresenter.java rename to linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/present/LogPresenter.java index 291d7314db..9710be95db 100644 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/present/LinkisLogPresenter.java +++ b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/present/LogPresenter.java @@ -17,31 +17,34 @@ package org.apache.linkis.cli.application.present; +import org.apache.linkis.cli.application.entity.present.Model; +import org.apache.linkis.cli.application.entity.present.Presenter; +import org.apache.linkis.cli.application.exception.PresenterException; +import org.apache.linkis.cli.application.exception.error.CommonErrMsg; +import org.apache.linkis.cli.application.exception.error.ErrorLevel; import org.apache.linkis.cli.application.observer.event.LinkisClientEvent; import org.apache.linkis.cli.application.observer.listener.LinkisClientListener; import org.apache.linkis.cli.application.present.model.LinkisLogModel; -import org.apache.linkis.cli.common.entity.present.Model; -import org.apache.linkis.cli.common.entity.present.PresentWay; -import org.apache.linkis.cli.common.entity.present.Presenter; -import org.apache.linkis.cli.common.exception.error.ErrorLevel; -import org.apache.linkis.cli.core.exception.PresenterException; -import org.apache.linkis.cli.core.exception.error.CommonErrMsg; -import org.apache.linkis.cli.core.present.PresentModeImpl; -import org.apache.linkis.cli.core.present.display.DisplayOperFactory; -import org.apache.linkis.cli.core.present.display.DisplayOperator; -import org.apache.linkis.cli.core.present.display.data.StdoutDisplayData; -import org.apache.linkis.cli.core.utils.CommonUtils; +import org.apache.linkis.cli.application.utils.CliUtils; +import org.apache.linkis.cli.application.utils.LoggerManager; import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -public class LinkisLogPresenter implements Presenter, LinkisClientListener { - private static Logger logger = LoggerFactory.getLogger(LinkisLogPresenter.class); +public class LogPresenter implements Presenter, LinkisClientListener { + private static Logger logger = LoggerFactory.getLogger(LogPresenter.class); @Override - public void present(Model model, PresentWay presentWay) { + public void update(LinkisClientEvent event, Object msg) { + Model model = new LinkisLogModel(); + model.buildModel(msg); + this.present(model); + } + + @Override + public void present(Model model) { if (!(model instanceof LinkisLogModel)) { throw new PresenterException( "PST0001", @@ -50,24 +53,16 @@ public void present(Model model, PresentWay presentWay) { "Input model for \"LinkisLogPresenter\" is not instance of \"LinkisJobIncLogModel\""); } LinkisLogModel logModel = (LinkisLogModel) model; - DisplayOperator displayOper = DisplayOperFactory.getDisplayOper(PresentModeImpl.STDOUT); while (!logModel.logFinReceived()) { String incLog = logModel.consumeLog(); if (StringUtils.isNotEmpty(incLog)) { - displayOper.doOutput(new StdoutDisplayData(incLog)); + LoggerManager.getPlaintTextLogger().info(incLog); } - CommonUtils.doSleepQuietly(500l); + CliUtils.doSleepQuietly(500l); } String incLog = logModel.consumeLog(); if (StringUtils.isNotEmpty(incLog)) { - displayOper.doOutput(new StdoutDisplayData(incLog)); + LoggerManager.getPlaintTextLogger().info(incLog); } } - - @Override - public void update(LinkisClientEvent event, Object msg) { - Model model = new LinkisLogModel(); - model.buildModel(msg); - this.present(model, null); - } } diff --git a/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/present/ResultPresenter.java b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/present/ResultPresenter.java new file mode 100644 index 0000000000..3c7281a7eb --- /dev/null +++ b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/present/ResultPresenter.java @@ -0,0 +1,301 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.cli.application.present; + +import org.apache.linkis.cli.application.constants.CliConstants; +import org.apache.linkis.cli.application.entity.present.Model; +import org.apache.linkis.cli.application.entity.present.Presenter; +import org.apache.linkis.cli.application.exception.PresenterException; +import org.apache.linkis.cli.application.exception.error.CommonErrMsg; +import org.apache.linkis.cli.application.exception.error.ErrorLevel; +import org.apache.linkis.cli.application.interactor.job.common.ResultSet; +import org.apache.linkis.cli.application.observer.event.LinkisClientEvent; +import org.apache.linkis.cli.application.observer.listener.LinkisClientListener; +import org.apache.linkis.cli.application.present.file.ResultFileWriter; +import org.apache.linkis.cli.application.present.model.LinkisResultModel; +import org.apache.linkis.cli.application.utils.CliUtils; +import org.apache.linkis.cli.application.utils.LoggerManager; + +import org.apache.commons.lang3.StringUtils; + +import java.text.MessageFormat; +import java.util.*; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class ResultPresenter implements Presenter, LinkisClientListener { + private static Logger logger = LoggerFactory.getLogger(ResultPresenter.class); + private Boolean writeToFile = false; + private String filePath = ""; + + public ResultPresenter() {} + + public ResultPresenter(Boolean writeToFile, String filePath) { + this.writeToFile = writeToFile; + this.filePath = filePath; + } + + @Override + public void update(LinkisClientEvent event, Object msg) { + Model model = new LinkisResultModel(); + model.buildModel(msg); + this.present(model); + } + + @Override + public void present(Model model) { + if (!(model instanceof LinkisResultModel)) { + throw new PresenterException( + "PST0001", + ErrorLevel.ERROR, + CommonErrMsg.PresenterErr, + "Input model for \"LinkisResultPresenter\" is not instance of \"LinkisResultModel\""); + } + LinkisResultModel resultModel = (LinkisResultModel) model; + + LoggerManager.getPlaintTextLogger().info(formatResultIndicator(resultModel)); + + if (!resultModel.getJobStatus().isJobSuccess()) { + LoggerManager.getInformationLogger() + .info("JobStatus is not \'success\'. Will not retrieve result-set."); + return; + } + String msg = ""; + if (resultModel.hasResult()) { + msg = + "Retrieving result-set, may take time if result-set is large, please do not exit program."; + } else { + msg = "Your job has no result."; + } + LoggerManager.getInformationLogger().info(msg); + + int preIdx = -1; + StringBuilder resultSb = new StringBuilder(); + + while (!resultModel.resultFinReceived()) { + preIdx = presentOneIteration(resultModel, preIdx, resultSb); + CliUtils.doSleepQuietly(500l); + } + presentOneIteration(resultModel, preIdx, resultSb); + + if (writeToFile) { + LoggerManager.getInformationLogger() + .info("ResultSet has been successfully written to path: " + filePath); + } + } + + protected int presentOneIteration( + LinkisResultModel resultModel, int preIdx, StringBuilder resultSb) { + List resultSets = resultModel.consumeResultContent(); + if (resultSets != null && !resultSets.isEmpty()) { + for (ResultSet c : resultSets) { + int idxResultset = c.getResultsetIdx(); + /** + * Notice: we assume result-sets are visited one by one in non-descending order!!! i.e. + * either idxResultset == preIdx or idxResultset - preIdx == 1 i.e. resultsets[0] -> + * resultsets[1] -> ... + */ + if (idxResultset - preIdx != 0 && idxResultset - preIdx != 1) { + throw new PresenterException( + "PST0002", + ErrorLevel.ERROR, + CommonErrMsg.PresenterErr, + "Linkis resultsets are visited in descending order or are not visited one-by-one"); + } + + boolean flag = idxResultset > preIdx; + if (idxResultset - preIdx == 1) { + resultSb.setLength(0); + resultSb + .append(MessageFormat.format(CliConstants.RESULTSET_LOGO, idxResultset + 1)) + .append(System.lineSeparator()); + if (c.getResultMeta() != null) { + resultSb.append(CliConstants.RESULTSET_META_BEGIN_LOGO).append(System.lineSeparator()); + resultSb.append(formatResultMeta(c.getResultMeta())); + resultSb.append(CliConstants.RESULTSET_META_END_LOGO).append(System.lineSeparator()); + } + } + preIdx = idxResultset; + String contentStr = formatResultContent(c.getResultMeta(), c.getContent()); + if (contentStr != null) { + resultSb.append(contentStr); + } + if (resultSb.length() != 0) { + if (writeToFile) { + String resultFileName = + resultModel.getUser() + + "-task-" + + resultModel.getJobID() + + "-result-" + + String.valueOf(idxResultset + 1) + + ".txt"; + ResultFileWriter.writeToFile(filePath, resultFileName, resultSb.toString(), flag); + } else { + LoggerManager.getPlaintTextLogger().info(resultSb.toString()); + } + resultSb.setLength(0); + } + } + } + return preIdx; + } + + protected String formatResultMeta(List> metaData) { + + StringBuilder outputBuilder = new StringBuilder(); + + if (metaData == null || metaData.size() == 0) { + return null; + } + + List titles = new ArrayList<>(); + + // gather keys as title + for (LinkedHashMap mapElement : metaData) { + if (mapElement == null || mapElement.size() == 0) { + continue; + } + + Set> entrySet = mapElement.entrySet(); + if (entrySet == null) { + break; + } + for (Map.Entry entry : entrySet) { + String key = entry.getKey(); + if (key != null && !titles.contains(key)) { + titles.add(key); + outputBuilder.append(key).append("\t"); + } + } + } + + outputBuilder.append(System.lineSeparator()); + + // gather value and print to output + for (LinkedHashMap mapElement : metaData) { + if (mapElement == null || mapElement.size() == 0) { + continue; + } + String candidate; + for (String title : titles) { + if (mapElement.containsKey(title)) { + candidate = mapElement.get(title); + } else { + candidate = "NULL"; + } + outputBuilder.append(candidate).append("\t"); + } + outputBuilder.append(System.lineSeparator()); + } + return outputBuilder.toString(); + } + + protected String formatResultContent( + List> metaData, List> contentData) { + + StringBuilder outputBuilder = new StringBuilder(); + if (contentData == null || contentData.size() == 0) { // finished + return null; + } + + int listLen = contentData.size(); + for (int i = 0; i < listLen; i++) { + List listElement = contentData.get(i); + if (listElement == null || listElement.size() == 0) { + continue; + } + for (String element : listElement) { + outputBuilder.append(element).append("\t"); + } + if (i < listLen - 1) { + outputBuilder.append(System.lineSeparator()); + } + } + + return outputBuilder.toString(); + } + + protected String formatResultIndicator(LinkisResultModel model) { + StringBuilder infoBuilder = new StringBuilder(); + String extraMsgStr = ""; + + if (model.getExtraMessage() != null) { + extraMsgStr = model.getExtraMessage(); + } + if (model.getJobStatus().isJobSuccess()) { + + LoggerManager.getInformationLogger() + .info("Job execute successfully! Will try get execute result"); + infoBuilder + .append("============Result:================") + .append(System.lineSeparator()) + .append("TaskId:") + .append(model.getJobID()) + .append(System.lineSeparator()) + .append("ExecId: ") + .append(model.getExecID()) + .append(System.lineSeparator()) + .append("User:") + .append(model.getUser()) + .append(System.lineSeparator()) + .append("Current job status:") + .append(model.getJobStatus()) + .append(System.lineSeparator()) + .append("extraMsg: ") + .append(extraMsgStr) + .append(System.lineSeparator()) + .append("result: ") + .append(extraMsgStr) + .append(System.lineSeparator()); + } else if (model.getJobStatus().isJobFinishedState()) { + LoggerManager.getInformationLogger().info("Job failed! Will not try get execute result."); + infoBuilder + .append("============Result:================") + .append(System.lineSeparator()) + .append("TaskId:") + .append(model.getJobID()) + .append(System.lineSeparator()) + .append("ExecId: ") + .append(model.getExecID()) + .append(System.lineSeparator()) + .append("User:") + .append(model.getUser()) + .append(System.lineSeparator()) + .append("Current job status:") + .append(model.getJobStatus()) + .append(System.lineSeparator()) + .append("extraMsg: ") + .append(extraMsgStr) + .append(System.lineSeparator()); + if (model.getErrCode() != null) { + infoBuilder.append("errCode: ").append(model.getErrCode()).append(System.lineSeparator()); + } + if (StringUtils.isNotBlank(model.getErrDesc())) { + infoBuilder.append("errDesc: ").append(model.getErrDesc()).append(System.lineSeparator()); + } + } else { + throw new PresenterException( + "PST0011", + ErrorLevel.ERROR, + CommonErrMsg.PresenterErr, + "Job is not completed but triggered ResultPresenter"); + } + return infoBuilder.toString(); + } +} diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/present/display/PlainTextFileWriter.java b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/present/file/ResultFileWriter.java similarity index 72% rename from linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/present/display/PlainTextFileWriter.java rename to linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/present/file/ResultFileWriter.java index 789b344501..9a54699165 100644 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/present/display/PlainTextFileWriter.java +++ b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/present/file/ResultFileWriter.java @@ -15,34 +15,21 @@ * limitations under the License. */ -package org.apache.linkis.cli.core.present.display; +package org.apache.linkis.cli.application.present.file; -import org.apache.linkis.cli.common.exception.error.ErrorLevel; -import org.apache.linkis.cli.core.exception.PresenterException; -import org.apache.linkis.cli.core.exception.error.CommonErrMsg; -import org.apache.linkis.cli.core.present.display.data.DisplayData; -import org.apache.linkis.cli.core.present.display.data.FileDisplayData; +import org.apache.linkis.cli.application.exception.PresenterException; +import org.apache.linkis.cli.application.exception.error.CommonErrMsg; +import org.apache.linkis.cli.application.exception.error.ErrorLevel; import java.io.BufferedWriter; import java.io.File; import java.io.FileOutputStream; import java.io.OutputStreamWriter; -public class PlainTextFileWriter implements DisplayOperator { - @Override - public void doOutput(DisplayData data) { - if (!(data instanceof FileDisplayData)) { - throw new PresenterException( - "PST0004", - ErrorLevel.ERROR, - CommonErrMsg.PresentDriverErr, - "input data is not instance of FileDisplayData"); - } +public class ResultFileWriter { - String pathName = ((FileDisplayData) data).getPathName(); - String fileName = pathName + File.separator + ((FileDisplayData) data).getFileName(); - String content = ((FileDisplayData) data).getContent(); - Boolean overWrite = ((FileDisplayData) data).getCreateNewFile(); + public static void writeToFile( + String pathName, String fileName, String content, Boolean overWrite) { File dir = new File(pathName); File file = new File(fileName); diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/present/model/HelpInfoModel.java b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/present/model/HelpInfoModel.java similarity index 77% rename from linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/present/model/HelpInfoModel.java rename to linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/present/model/HelpInfoModel.java index c2b5f5e772..46dcec2d5e 100644 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/present/model/HelpInfoModel.java +++ b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/present/model/HelpInfoModel.java @@ -15,13 +15,13 @@ * limitations under the License. */ -package org.apache.linkis.cli.core.present.model; +package org.apache.linkis.cli.application.present.model; -import org.apache.linkis.cli.common.entity.command.CmdTemplate; -import org.apache.linkis.cli.common.entity.present.Model; -import org.apache.linkis.cli.common.exception.error.ErrorLevel; -import org.apache.linkis.cli.core.exception.TransformerException; -import org.apache.linkis.cli.core.exception.error.CommonErrMsg; +import org.apache.linkis.cli.application.entity.command.CmdTemplate; +import org.apache.linkis.cli.application.entity.present.Model; +import org.apache.linkis.cli.application.exception.TransformerException; +import org.apache.linkis.cli.application.exception.error.CommonErrMsg; +import org.apache.linkis.cli.application.exception.error.ErrorLevel; public class HelpInfoModel implements Model { CmdTemplate template; diff --git a/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/present/model/LinkisJobInfoModel.java b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/present/model/LinkisJobInfoModel.java new file mode 100644 index 0000000000..b428a788dd --- /dev/null +++ b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/present/model/LinkisJobInfoModel.java @@ -0,0 +1,89 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.cli.application.present.model; + +import org.apache.linkis.cli.application.entity.job.JobStatus; +import org.apache.linkis.cli.application.entity.present.Model; +import org.apache.linkis.cli.application.exception.TransformerException; +import org.apache.linkis.cli.application.exception.error.CommonErrMsg; +import org.apache.linkis.cli.application.exception.error.ErrorLevel; +import org.apache.linkis.cli.application.operator.ujes.LinkisOperResultAdapter; + +import java.util.Date; + +public class LinkisJobInfoModel implements Model { + + private String cid; + private String jobId; + private String message; + private String exception; + private String cause; + + private String taskID; + private String instance; + private String simpleExecId; + private String execId; + private String umUser; + private String executionCode; + private String logPath; + private JobStatus status; + private String engineType; + private String runType; + private Long costTime; + private Date createdTime; + private Date updatedTime; + private Date engineStartTime; + private Integer errCode; + private String errMsg; + private String executeApplicationName; + private String requestApplicationName; + private Float progress; + + @Override + public void buildModel(Object data) { + if (!(data instanceof LinkisOperResultAdapter)) { + throw new TransformerException( + "TFM0010", + ErrorLevel.ERROR, + CommonErrMsg.TransformerException, + "Failed to init LinkisJobInfoModel: " + + data.getClass().getCanonicalName() + + "is not instance of \"LinkisOperResultAdapter\""); + } + this.jobId = ((LinkisOperResultAdapter) data).getJobID(); + this.taskID = ((LinkisOperResultAdapter) data).getJobID(); + this.instance = ((LinkisOperResultAdapter) data).getInstance(); + this.simpleExecId = ((LinkisOperResultAdapter) data).getSimpleExecId(); + this.execId = ((LinkisOperResultAdapter) data).getStrongerExecId(); + this.umUser = ((LinkisOperResultAdapter) data).getUmUser(); + this.executionCode = ((LinkisOperResultAdapter) data).getExecutionCode(); + this.logPath = ((LinkisOperResultAdapter) data).getLogPath(); + this.status = ((LinkisOperResultAdapter) data).getJobStatus(); + this.engineType = ((LinkisOperResultAdapter) data).getEngineType(); + this.runType = ((LinkisOperResultAdapter) data).getRunType(); + this.costTime = ((LinkisOperResultAdapter) data).getCostTime(); + this.createdTime = ((LinkisOperResultAdapter) data).getCreatedTime(); + this.updatedTime = ((LinkisOperResultAdapter) data).getUpdatedTime(); + this.engineStartTime = ((LinkisOperResultAdapter) data).getEngineStartTime(); + this.errCode = ((LinkisOperResultAdapter) data).getErrCode(); + this.errMsg = ((LinkisOperResultAdapter) data).getErrDesc(); + this.executeApplicationName = ((LinkisOperResultAdapter) data).getExecuteApplicationName(); + this.requestApplicationName = ((LinkisOperResultAdapter) data).getRequestApplicationName(); + this.progress = ((LinkisOperResultAdapter) data).getJobProgress(); + } +} diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/present/model/LinkisLogModel.java b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/present/model/LinkisLogModel.java similarity index 74% rename from linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/present/model/LinkisLogModel.java rename to linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/present/model/LinkisLogModel.java index 503c778339..e580547887 100644 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/present/model/LinkisLogModel.java +++ b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/present/model/LinkisLogModel.java @@ -17,18 +17,18 @@ package org.apache.linkis.cli.application.present.model; -import org.apache.linkis.cli.application.interactor.job.data.LinkisLogData; -import org.apache.linkis.cli.common.entity.present.Model; -import org.apache.linkis.cli.common.exception.error.ErrorLevel; -import org.apache.linkis.cli.core.exception.TransformerException; -import org.apache.linkis.cli.core.exception.error.CommonErrMsg; +import org.apache.linkis.cli.application.entity.present.Model; +import org.apache.linkis.cli.application.exception.TransformerException; +import org.apache.linkis.cli.application.exception.error.CommonErrMsg; +import org.apache.linkis.cli.application.exception.error.ErrorLevel; +import org.apache.linkis.cli.application.interactor.job.common.LogData; public class LinkisLogModel implements Model { - private LinkisLogData data; + private LogData data; @Override public void buildModel(Object data) { - if (!(data instanceof LinkisLogData)) { + if (!(data instanceof LogData)) { throw new TransformerException( "TFM0010", ErrorLevel.ERROR, @@ -37,7 +37,7 @@ public void buildModel(Object data) { + data.getClass().getCanonicalName() + "is not instance of \"LinkisLogData\""); } - this.data = (LinkisLogData) data; + this.data = (LogData) data; } public String consumeLog() { @@ -45,6 +45,6 @@ public String consumeLog() { } public boolean logFinReceived() { - return data.logFinReceived(); + return data.isLogFin(); } } diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/present/model/LinkisResultModel.java b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/present/model/LinkisResultModel.java similarity index 63% rename from linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/present/model/LinkisResultModel.java rename to linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/present/model/LinkisResultModel.java index 1dce057c3e..fd9e8eb9a2 100644 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/present/model/LinkisResultModel.java +++ b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/present/model/LinkisResultModel.java @@ -17,22 +17,22 @@ package org.apache.linkis.cli.application.present.model; -import org.apache.linkis.cli.application.interactor.job.data.LinkisResultData; -import org.apache.linkis.cli.application.interactor.job.data.LinkisResultSet; -import org.apache.linkis.cli.common.entity.job.JobStatus; -import org.apache.linkis.cli.common.entity.present.Model; -import org.apache.linkis.cli.common.exception.error.ErrorLevel; -import org.apache.linkis.cli.core.exception.TransformerException; -import org.apache.linkis.cli.core.exception.error.CommonErrMsg; +import org.apache.linkis.cli.application.entity.job.JobStatus; +import org.apache.linkis.cli.application.entity.present.Model; +import org.apache.linkis.cli.application.exception.TransformerException; +import org.apache.linkis.cli.application.exception.error.CommonErrMsg; +import org.apache.linkis.cli.application.exception.error.ErrorLevel; +import org.apache.linkis.cli.application.interactor.job.common.ResultData; +import org.apache.linkis.cli.application.interactor.job.common.ResultSet; import java.util.List; public class LinkisResultModel implements Model { - private LinkisResultData data; + private ResultData data; @Override public void buildModel(Object data) { - if (!(data instanceof LinkisResultData)) { + if (!(data instanceof ResultData)) { throw new TransformerException( "TFM0010", ErrorLevel.ERROR, @@ -41,15 +41,15 @@ public void buildModel(Object data) { + data.getClass().getCanonicalName() + "is not instance of \"LinkisResultData\""); } - this.data = (LinkisResultData) data; + this.data = (ResultData) data; } - public List consumeResultContent() { + public List consumeResultContent() { return data.consumeResultContent(); } public boolean resultFinReceived() { - return data.resultFinReceived(); + return data.isResultFin(); } public JobStatus getJobStatus() { @@ -67,4 +67,20 @@ public String getUser() { public boolean hasResult() { return data.hasResult(); } + + public String getExecID() { + return data.getExecID(); + } + + public Integer getErrCode() { + return data.getErrCode(); + } + + public String getErrDesc() { + return data.getErrDesc(); + } + + public String getExtraMessage() { + return data.getExtraMessage(); + } } diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/utils/ExecutionUtils.java b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/utils/CliUtils.java similarity index 65% rename from linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/utils/ExecutionUtils.java rename to linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/utils/CliUtils.java index 82a3d8bff2..3f8d86d48e 100644 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/main/java/org/apache/linkis/cli/application/utils/ExecutionUtils.java +++ b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/utils/CliUtils.java @@ -17,31 +17,38 @@ package org.apache.linkis.cli.application.utils; -import org.apache.linkis.cli.application.constants.AppKeys; +import org.apache.linkis.cli.application.constants.CliKeys; import org.apache.linkis.cli.application.constants.LinkisConstants; -import org.apache.linkis.cli.common.entity.var.VarAccess; -import org.apache.linkis.cli.common.exception.error.ErrorLevel; -import org.apache.linkis.cli.core.exception.BuilderException; -import org.apache.linkis.cli.core.exception.error.CommonErrMsg; -import org.apache.linkis.cli.core.utils.LogUtils; +import org.apache.linkis.cli.application.entity.var.VarAccess; +import org.apache.linkis.cli.application.exception.BuilderException; +import org.apache.linkis.cli.application.exception.error.CommonErrMsg; +import org.apache.linkis.cli.application.exception.error.ErrorLevel; import org.apache.commons.lang3.StringUtils; import java.io.*; +import java.util.HashMap; +import java.util.Map; import java.util.Set; -public class ExecutionUtils { +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; + +public class CliUtils { + public static final Gson GSON = + new GsonBuilder().setPrettyPrinting().disableHtmlEscaping().create(); + public static String getSubmitUser(VarAccess stdVarAccess, String osUser, Set adminSet) { String enableSpecifyUserStr = stdVarAccess.getVar( - String.class, AppKeys.LINKIS_CLIENT_NONCUSTOMIZABLE_ENABLE_USER_SPECIFICATION); + String.class, CliKeys.LINKIS_CLIENT_NONCUSTOMIZABLE_ENABLE_USER_SPECIFICATION); Boolean enableSpecifyUser = Boolean.parseBoolean(enableSpecifyUserStr) || adminSet.contains(osUser); String authenticationStrategy = stdVarAccess.getVarOrDefault( String.class, - AppKeys.LINKIS_COMMON_AUTHENTICATION_STRATEGY, + CliKeys.LINKIS_COMMON_AUTHENTICATION_STRATEGY, LinkisConstants.AUTH_STRATEGY_STATIC); String submitUsr; @@ -53,8 +60,8 @@ public static String getSubmitUser(VarAccess stdVarAccess, String osUser, Set adminSet) { String enableSpecifyPRoxyUserStr = - stdVarAccess.getVar(String.class, AppKeys.LINKIS_CLIENT_NONCUSTOMIZABLE_ENABLE_PROXY_USER); + stdVarAccess.getVar(String.class, CliKeys.LINKIS_CLIENT_NONCUSTOMIZABLE_ENABLE_PROXY_USER); Boolean enableSpecifyProxyUser = Boolean.parseBoolean(enableSpecifyPRoxyUserStr) || adminSet.contains(submitUsr); @@ -129,8 +136,8 @@ public static String getProxyUser( String proxyUsr; if (enableSpecifyProxyUser) { - if (stdVarAccess.hasVar(AppKeys.JOB_COMMON_PROXY_USER)) { - proxyUsr = stdVarAccess.getVar(String.class, AppKeys.JOB_COMMON_PROXY_USER); + if (stdVarAccess.hasVar(CliKeys.JOB_COMMON_PROXY_USER)) { + proxyUsr = stdVarAccess.getVar(String.class, CliKeys.JOB_COMMON_PROXY_USER); if (!adminSet.contains(submitUsr) && adminSet.contains(proxyUsr)) { throw new BuilderException( "BLD0010", @@ -140,14 +147,14 @@ public static String getProxyUser( } } else { proxyUsr = submitUsr; - LogUtils.getInformationLogger() + LoggerManager.getInformationLogger() .info( "user does not specify proxy-user, will use current submit-user \"" + submitUsr + "\" by default."); } - } else if (stdVarAccess.hasVar(AppKeys.JOB_COMMON_PROXY_USER)) { - proxyUsr = stdVarAccess.getVar(String.class, AppKeys.JOB_COMMON_PROXY_USER); + } else if (stdVarAccess.hasVar(CliKeys.JOB_COMMON_PROXY_USER)) { + proxyUsr = stdVarAccess.getVar(String.class, CliKeys.JOB_COMMON_PROXY_USER); if (!StringUtils.equals(proxyUsr, submitUsr)) { throw new BuilderException( "BLD0010", @@ -157,7 +164,7 @@ public static String getProxyUser( } } else { proxyUsr = submitUsr; - LogUtils.getInformationLogger() + LoggerManager.getInformationLogger() .info( "user does not specify proxy-user, will use current submit-user \"" + proxyUsr @@ -199,4 +206,66 @@ public static String readFile(String path) { e); } } + + public static T castStringToAny(Class clazz, String val) { + if (StringUtils.isBlank(val)) { + return null; + } + T ret = null; + if (clazz == Object.class) { + ret = clazz.cast(val); + } else if (clazz == String.class) { + ret = clazz.cast(val); + } else if (clazz == Integer.class) { + ret = clazz.cast(Integer.parseInt(val)); + } else if (clazz == Double.class) { + ret = clazz.cast(Double.parseDouble(val)); + } else if (clazz == Float.class) { + ret = clazz.cast(Float.parseFloat(val)); + } else if (clazz == Long.class) { + ret = clazz.cast(Long.parseLong(val)); + } else if (clazz == Boolean.class) { + ret = clazz.cast(Boolean.parseBoolean(val)); + } + return ret; + } + + public static Map parseKVStringToMap(String kvStr, String separator) { + if (StringUtils.isBlank(separator)) { + separator = ","; + } + if (StringUtils.isBlank(kvStr)) { + return null; + } + Map argsProps = new HashMap<>(); + String[] args = StringUtils.splitByWholeSeparator(kvStr, separator); + for (String arg : args) { + int index = arg.indexOf("="); + if (index != -1) { + argsProps.put(arg.substring(0, index).trim(), arg.substring(index + 1).trim()); + } + } + + return argsProps; + } + + public static boolean isValidExecId(String execId) { + boolean ret = false; + if (StringUtils.isNotBlank(execId)) { + ret = true; + } + return ret; + } + + public static String progressInPercentage(float progress) { + return String.valueOf(progress * 100) + "%"; + } + + public static void doSleepQuietly(Long sleepMills) { + try { + Thread.sleep(sleepMills); + } catch (Exception ignore) { + // ignored + } + } } diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/utils/LogUtils.java b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/utils/LoggerManager.java similarity index 93% rename from linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/utils/LogUtils.java rename to linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/utils/LoggerManager.java index b8147a4a60..bf302e9e85 100644 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/utils/LogUtils.java +++ b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/utils/LoggerManager.java @@ -15,12 +15,12 @@ * limitations under the License. */ -package org.apache.linkis.cli.core.utils; +package org.apache.linkis.cli.application.utils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -public class LogUtils { +public class LoggerManager { private static final Logger PlainTextLogger = LoggerFactory.getLogger("PlaintTextLogger"); private static final Logger InformationLogger = LoggerFactory.getLogger("InformationLogger"); diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/utils/SchedulerUtils.java b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/utils/SchedulerManager.java similarity index 94% rename from linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/utils/SchedulerUtils.java rename to linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/utils/SchedulerManager.java index 5cfce48628..48aa367959 100644 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-core/src/main/java/org/apache/linkis/cli/core/utils/SchedulerUtils.java +++ b/linkis-computation-governance/linkis-client/linkis-cli/src/main/java/org/apache/linkis/cli/application/utils/SchedulerManager.java @@ -15,12 +15,12 @@ * limitations under the License. */ -package org.apache.linkis.cli.core.utils; +package org.apache.linkis.cli.application.utils; import java.util.concurrent.*; import java.util.concurrent.atomic.AtomicInteger; -public class SchedulerUtils { +public class SchedulerManager { private static ExecutorService fixedThreadPool; private static ThreadPoolExecutor cachedThreadPool; private static int THREAD_NUM = 5; @@ -62,7 +62,7 @@ public static ExecutorService newFixedThreadPool( public static ThreadPoolExecutor getCachedThreadPoolExecutor() { if (cachedThreadPool == null) { - synchronized (SchedulerUtils.class) { + synchronized (SchedulerManager.class) { if (cachedThreadPool == null) { cachedThreadPool = newCachedThreadPool(THREAD_NUM, THREAD_NAME, IS_DEAMON); } @@ -73,7 +73,7 @@ public static ThreadPoolExecutor getCachedThreadPoolExecutor() { public static ExecutorService getFixedThreadPool() { if (fixedThreadPool == null) { - synchronized (SchedulerUtils.class) { + synchronized (SchedulerManager.class) { if (fixedThreadPool == null) { fixedThreadPool = newFixedThreadPool(THREAD_NUM, THREAD_NAME, IS_DEAMON); } diff --git a/linkis-computation-governance/linkis-client/linkis-cli/src/main/resources/version.properties b/linkis-computation-governance/linkis-client/linkis-cli/src/main/resources/version.properties new file mode 100644 index 0000000000..0888f0b359 --- /dev/null +++ b/linkis-computation-governance/linkis-client/linkis-cli/src/main/resources/version.properties @@ -0,0 +1 @@ +cli.version=${project.version} \ No newline at end of file diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/test/java/org/apache/linkis/cli/application/LinkisClientApplicationTest.java b/linkis-computation-governance/linkis-client/linkis-cli/src/test/java/org/apache/linkis/cli/application/LinkisClientApplicationTest.java similarity index 88% rename from linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/test/java/org/apache/linkis/cli/application/LinkisClientApplicationTest.java rename to linkis-computation-governance/linkis-client/linkis-cli/src/test/java/org/apache/linkis/cli/application/LinkisClientApplicationTest.java index 0af2226661..d7ca2c0349 100644 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/test/java/org/apache/linkis/cli/application/LinkisClientApplicationTest.java +++ b/linkis-computation-governance/linkis-client/linkis-cli/src/test/java/org/apache/linkis/cli/application/LinkisClientApplicationTest.java @@ -28,26 +28,36 @@ public class LinkisClientApplicationTest { String[] cmdStr; String[] cmdStr2; + String[] cmdStr3; @BeforeEach public void before() { System.setProperty("conf.root", "src/test/resources/conf/"); System.setProperty("user.name", "hadoop"); + cmdStr3 = new String[] {"--version"}; cmdStr2 = new String[] { - // "--gatewayUrl", "http://127.0.0.1:8090", - // "--authStg", "token", - // "--authKey", "Validation-Code", - // "--authVal", "BML-AUTH", - // "job", - // "kill", - // "-j", "1121", - // "-submitUser", "user", - // "-proxyUser", "user", - - // "-varMap", "name=\"tables\"", - // "-varMap", "name=\"databases\"" - + "--gatewayUrl", + "http://127.0.0.1:9001", + "--authStg", + "token", + "--authKey", + "Validation-Code", + "--authVal", + "BML-AUTH", + "--status", + // "--log", + // "--kill", + // "--result", + "5773107", + "-submitUser", + "hadoop", + "-proxyUser", + "hadoop", + "-varMap", + "name=\"tables\"", + "-varMap", + "name2=\"databases\"" }; cmdStr = new String[] { @@ -99,6 +109,7 @@ public void before() { "-codeType", "shell", "-code", + // "exit -1", "whoami", // "-engineType", "spark-2.4.3", @@ -165,7 +176,8 @@ public void testProcessInput() { /** Method: exec(ProcessedData data) */ @Test public void testExec() { - // LinkisClientApplication.main(cmdStr); + // LinkisClientApplication.main(cmdStr); + LinkisClientApplication.main(cmdStr3); // LinkisClientApplication.main(cmdStr2); /* try { diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/test/java/org/apache/linkis/cli/application/constants/AppConstantsTest.java b/linkis-computation-governance/linkis-client/linkis-cli/src/test/java/org/apache/linkis/cli/application/constants/CliConstantsTest.java similarity index 71% rename from linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/test/java/org/apache/linkis/cli/application/constants/AppConstantsTest.java rename to linkis-computation-governance/linkis-client/linkis-cli/src/test/java/org/apache/linkis/cli/application/constants/CliConstantsTest.java index bb16d43e47..b576c89f4f 100644 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/test/java/org/apache/linkis/cli/application/constants/AppConstantsTest.java +++ b/linkis-computation-governance/linkis-client/linkis-cli/src/test/java/org/apache/linkis/cli/application/constants/CliConstantsTest.java @@ -21,25 +21,25 @@ import org.junit.jupiter.api.DisplayName; import org.junit.jupiter.api.Test; -public class AppConstantsTest { +public class CliConstantsTest { @Test @DisplayName("constTest") public void constTest() { - String defaultConfigName = AppConstants.DEFAULT_CONFIG_NAME; - Long jobQuerySleepMills = AppConstants.JOB_QUERY_SLEEP_MILLS; - String resultsetLogo = AppConstants.RESULTSET_LOGO; - String resultsetMetaBeginLogo = AppConstants.RESULTSET_META_BEGIN_LOGO; - String resultsetMetaEndLogo = AppConstants.RESULTSET_META_END_LOGO; - String resultsetSeparatorLogo = AppConstants.RESULTSET_SEPARATOR_LOGO; - Integer resultsetPageSize = AppConstants.RESULTSET_PAGE_SIZE; - String jobCreatorDefault = AppConstants.JOB_CREATOR_DEFAULT; - String jobCreatorAsyncDefault = AppConstants.JOB_CREATOR_ASYNC_DEFAULT; - String dummyCid = AppConstants.DUMMY_CID; - String linkisCli = AppConstants.LINKIS_CLI; - String ujesMode = AppConstants.UJES_MODE; - String onceMode = AppConstants.ONCE_MODE; + String defaultConfigName = CliConstants.DEFAULT_CONFIG_NAME; + Long jobQuerySleepMills = CliConstants.JOB_QUERY_SLEEP_MILLS; + String resultsetLogo = CliConstants.RESULTSET_LOGO; + String resultsetMetaBeginLogo = CliConstants.RESULTSET_META_BEGIN_LOGO; + String resultsetMetaEndLogo = CliConstants.RESULTSET_META_END_LOGO; + String resultsetSeparatorLogo = CliConstants.RESULTSET_SEPARATOR_LOGO; + Integer resultsetPageSize = CliConstants.RESULTSET_PAGE_SIZE; + String jobCreatorDefault = CliConstants.JOB_CREATOR_DEFAULT; + String jobCreatorAsyncDefault = CliConstants.JOB_CREATOR_ASYNC_DEFAULT; + String dummyCid = CliConstants.DUMMY_CID; + String linkisCli = CliConstants.LINKIS_CLI; + String ujesMode = CliConstants.UJES_MODE; + String onceMode = CliConstants.ONCE_MODE; Assertions.assertEquals("linkis-cli.properties", defaultConfigName); Assertions.assertTrue(2000L == jobQuerySleepMills.longValue()); diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/test/java/org/apache/linkis/cli/application/constants/AppKeysTest.java b/linkis-computation-governance/linkis-client/linkis-cli/src/test/java/org/apache/linkis/cli/application/constants/CliKeysTest.java similarity index 73% rename from linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/test/java/org/apache/linkis/cli/application/constants/AppKeysTest.java rename to linkis-computation-governance/linkis-client/linkis-cli/src/test/java/org/apache/linkis/cli/application/constants/CliKeysTest.java index f2477149b8..3fc7b84a0a 100644 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/test/java/org/apache/linkis/cli/application/constants/AppKeysTest.java +++ b/linkis-computation-governance/linkis-client/linkis-cli/src/test/java/org/apache/linkis/cli/application/constants/CliKeysTest.java @@ -21,27 +21,27 @@ import org.junit.jupiter.api.DisplayName; import org.junit.jupiter.api.Test; -public class AppKeysTest { +public class CliKeysTest { @Test @DisplayName("constTest") public void constTest() { - String adminUsers = AppKeys.ADMIN_USERS; - String linkisClientNoncustomizable = AppKeys.LINKIS_CLIENT_NONCUSTOMIZABLE; - String logPathKey = AppKeys.LOG_PATH_KEY; - String logFileKey = AppKeys.LOG_FILE_KEY; - String clientConfigRootKey = AppKeys.CLIENT_CONFIG_ROOT_KEY; - String defaultConfigFileNameKey = AppKeys.DEFAULT_CONFIG_FILE_NAME_KEY; - String linuxUserKey = AppKeys.LINUX_USER_KEY; - String jobExec = AppKeys.JOB_EXEC; - String jobExecCode = AppKeys.JOB_EXEC_CODE; - String jobContent = AppKeys.JOB_CONTENT; - String jobSource = AppKeys.JOB_SOURCE; - String jobParamConf = AppKeys.JOB_PARAM_CONF; - String jobParamRuntime = AppKeys.JOB_PARAM_RUNTIME; - String jobParamVar = AppKeys.JOB_PARAM_VAR; - String jobLabel = AppKeys.JOB_LABEL; + String adminUsers = CliKeys.ADMIN_USERS; + String linkisClientNoncustomizable = CliKeys.LINKIS_CLIENT_NONCUSTOMIZABLE; + String logPathKey = CliKeys.LOG_PATH_KEY; + String logFileKey = CliKeys.LOG_FILE_KEY; + String clientConfigRootKey = CliKeys.CLIENT_CONFIG_ROOT_KEY; + String defaultConfigFileNameKey = CliKeys.DEFAULT_CONFIG_FILE_NAME_KEY; + String linuxUserKey = CliKeys.LINUX_USER_KEY; + String jobExec = CliKeys.JOB_EXEC; + String jobExecCode = CliKeys.JOB_EXEC_CODE; + String jobContent = CliKeys.JOB_CONTENT; + String jobSource = CliKeys.JOB_SOURCE; + String jobParamConf = CliKeys.JOB_PARAM_CONF; + String jobParamRuntime = CliKeys.JOB_PARAM_RUNTIME; + String jobParamVar = CliKeys.JOB_PARAM_VAR; + String jobLabel = CliKeys.JOB_LABEL; Assertions.assertEquals("hadoop,root,shangda", adminUsers); Assertions.assertEquals("wds.linkis.client.noncustomizable", linkisClientNoncustomizable); diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/test/java/org/apache/linkis/cli/application/constants/LinkisConstantsTest.java b/linkis-computation-governance/linkis-client/linkis-cli/src/test/java/org/apache/linkis/cli/application/constants/LinkisConstantsTest.java similarity index 100% rename from linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/test/java/org/apache/linkis/cli/application/constants/LinkisConstantsTest.java rename to linkis-computation-governance/linkis-client/linkis-cli/src/test/java/org/apache/linkis/cli/application/constants/LinkisConstantsTest.java diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/test/java/org/apache/linkis/cli/application/constants/LinkisKeysTest.java b/linkis-computation-governance/linkis-client/linkis-cli/src/test/java/org/apache/linkis/cli/application/constants/LinkisKeysTest.java similarity index 100% rename from linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/test/java/org/apache/linkis/cli/application/constants/LinkisKeysTest.java rename to linkis-computation-governance/linkis-client/linkis-cli/src/test/java/org/apache/linkis/cli/application/constants/LinkisKeysTest.java diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/test/java/org/apache/linkis/cli/application/constants/TestConstants.java b/linkis-computation-governance/linkis-client/linkis-cli/src/test/java/org/apache/linkis/cli/application/constants/TestConstants.java similarity index 100% rename from linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/test/java/org/apache/linkis/cli/application/constants/TestConstants.java rename to linkis-computation-governance/linkis-client/linkis-cli/src/test/java/org/apache/linkis/cli/application/constants/TestConstants.java diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/test/java/org/apache/linkis/cli/application/interactor/command/LinkisCmdTypeTest.java b/linkis-computation-governance/linkis-client/linkis-cli/src/test/java/org/apache/linkis/cli/application/interactor/command/CliCmdTypeTest.java similarity index 86% rename from linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/test/java/org/apache/linkis/cli/application/interactor/command/LinkisCmdTypeTest.java rename to linkis-computation-governance/linkis-client/linkis-cli/src/test/java/org/apache/linkis/cli/application/interactor/command/CliCmdTypeTest.java index 16f47274d4..fecda11db7 100644 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/test/java/org/apache/linkis/cli/application/interactor/command/LinkisCmdTypeTest.java +++ b/linkis-computation-governance/linkis-client/linkis-cli/src/test/java/org/apache/linkis/cli/application/interactor/command/CliCmdTypeTest.java @@ -21,15 +21,15 @@ import org.junit.jupiter.api.DisplayName; import org.junit.jupiter.api.Test; -public class LinkisCmdTypeTest { +public class CliCmdTypeTest { @Test @DisplayName("enumTest") public void enumTest() { - int id = LinkisCmdType.UNIVERSAL.getId(); - String name = LinkisCmdType.UNIVERSAL.getName(); - String desc = LinkisCmdType.UNIVERSAL.getDesc(); + int id = CliCmdType.UNIVERSAL.getId(); + String name = CliCmdType.UNIVERSAL.getName(); + String desc = CliCmdType.UNIVERSAL.getDesc(); Assertions.assertTrue(1 == id); Assertions.assertEquals("linkis-cli", name); diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/test/java/org/apache/linkis/cli/application/interactor/command/template/TestCmdType.java b/linkis-computation-governance/linkis-client/linkis-cli/src/test/java/org/apache/linkis/cli/application/interactor/command/template/TestCmdType.java similarity index 96% rename from linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/test/java/org/apache/linkis/cli/application/interactor/command/template/TestCmdType.java rename to linkis-computation-governance/linkis-client/linkis-cli/src/test/java/org/apache/linkis/cli/application/interactor/command/template/TestCmdType.java index 54a5952f09..0155ee5029 100644 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/test/java/org/apache/linkis/cli/application/interactor/command/template/TestCmdType.java +++ b/linkis-computation-governance/linkis-client/linkis-cli/src/test/java/org/apache/linkis/cli/application/interactor/command/template/TestCmdType.java @@ -18,7 +18,7 @@ package org.apache.linkis.cli.application.interactor.command.template; import org.apache.linkis.cli.application.constants.TestConstants; -import org.apache.linkis.cli.common.entity.command.CmdType; +import org.apache.linkis.cli.application.entity.command.CmdType; public enum TestCmdType implements CmdType { TEST_PRIMARY(TestConstants.PRIMARY_COMMAND, 1, TestConstants.SPARK_DESC), diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/test/java/org/apache/linkis/cli/application/interactor/command/template/TestParamMapper.java b/linkis-computation-governance/linkis-client/linkis-cli/src/test/java/org/apache/linkis/cli/application/interactor/command/template/TestParamMapper.java similarity index 78% rename from linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/test/java/org/apache/linkis/cli/application/interactor/command/template/TestParamMapper.java rename to linkis-computation-governance/linkis-client/linkis-cli/src/test/java/org/apache/linkis/cli/application/interactor/command/template/TestParamMapper.java index 1a8959f097..a077cfcca7 100644 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/test/java/org/apache/linkis/cli/application/interactor/command/template/TestParamMapper.java +++ b/linkis-computation-governance/linkis-client/linkis-cli/src/test/java/org/apache/linkis/cli/application/interactor/command/template/TestParamMapper.java @@ -17,19 +17,19 @@ package org.apache.linkis.cli.application.interactor.command.template; -import org.apache.linkis.cli.application.constants.AppKeys; +import org.apache.linkis.cli.application.constants.CliKeys; import org.apache.linkis.cli.application.constants.TestConstants; -import org.apache.linkis.cli.core.interactor.command.parser.transformer.ParamKeyMapper; +import org.apache.linkis.cli.application.interactor.command.parser.transformer.ParamKeyMapper; public class TestParamMapper extends ParamKeyMapper { @Override public void initMapperRules() { - super.updateMapping(TestConstants.PARAM_COMMON_CMD, AppKeys.JOB_EXEC_CODE); - super.updateMapping(TestConstants.PARAM_PROXY, AppKeys.LINKIS_COMMON_GATEWAY_URL); - super.updateMapping(TestConstants.PARAM_USER, AppKeys.LINKIS_COMMON_TOKEN_KEY); - super.updateMapping(TestConstants.PARAM_USR_CONF, AppKeys.LINKIS_CLIENT_USER_CONFIG); - super.updateMapping(TestConstants.PARAM_PASSWORD, AppKeys.LINKIS_COMMON_TOKEN_VALUE); - super.updateMapping(TestConstants.PARAM_PROXY_USER, AppKeys.JOB_COMMON_PROXY_USER); + super.updateMapping(TestConstants.PARAM_COMMON_CMD, CliKeys.JOB_EXEC_CODE); + super.updateMapping(TestConstants.PARAM_PROXY, CliKeys.LINKIS_COMMON_GATEWAY_URL); + super.updateMapping(TestConstants.PARAM_USER, CliKeys.LINKIS_COMMON_TOKEN_KEY); + super.updateMapping(TestConstants.PARAM_USR_CONF, CliKeys.LINKIS_CLIENT_USER_CONFIG); + super.updateMapping(TestConstants.PARAM_PASSWORD, CliKeys.LINKIS_COMMON_TOKEN_VALUE); + super.updateMapping(TestConstants.PARAM_PROXY_USER, CliKeys.JOB_COMMON_PROXY_USER); updateMapping( TestConstants.PARAM_SPARK_EXECUTOR_CORES, TestConstants.LINKIS_SPARK_EXECUTOR_CORES); @@ -43,7 +43,7 @@ public void initMapperRules() { updateMapping( TestConstants.PARAM_SPARK_SHUFFLE_PARTITIONS, TestConstants.LINKIS_SPARK_SHUFFLE_PARTITIONS); - updateMapping(TestConstants.PARAM_SPARK_RUNTYPE, AppKeys.JOB_LABEL_CODE_TYPE); + updateMapping(TestConstants.PARAM_SPARK_RUNTYPE, CliKeys.JOB_LABEL_CODE_TYPE); updateMapping(TestConstants.PARAM_YARN_QUEUE, TestConstants.YARN_QUEUE); } diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/test/java/org/apache/linkis/cli/application/interactor/command/template/TestSparkCmdTemplate.java b/linkis-computation-governance/linkis-client/linkis-cli/src/test/java/org/apache/linkis/cli/application/interactor/command/template/TestSparkCmdTemplate.java similarity index 94% rename from linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/test/java/org/apache/linkis/cli/application/interactor/command/template/TestSparkCmdTemplate.java rename to linkis-computation-governance/linkis-client/linkis-cli/src/test/java/org/apache/linkis/cli/application/interactor/command/template/TestSparkCmdTemplate.java index c9274fbf71..fb53803b6a 100644 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/test/java/org/apache/linkis/cli/application/interactor/command/template/TestSparkCmdTemplate.java +++ b/linkis-computation-governance/linkis-client/linkis-cli/src/test/java/org/apache/linkis/cli/application/interactor/command/template/TestSparkCmdTemplate.java @@ -17,11 +17,10 @@ package org.apache.linkis.cli.application.interactor.command.template; -import org.apache.linkis.cli.application.constants.AppKeys; +import org.apache.linkis.cli.application.constants.CliKeys; import org.apache.linkis.cli.application.constants.TestConstants; -import org.apache.linkis.cli.core.exception.CommandException; -import org.apache.linkis.cli.core.interactor.command.template.AbstractCmdTemplate; -import org.apache.linkis.cli.core.interactor.command.template.option.StdOption; +import org.apache.linkis.cli.application.exception.CommandException; +import org.apache.linkis.cli.application.interactor.command.template.option.StdOption; import java.util.HashMap; import java.util.Map; @@ -50,7 +49,7 @@ public class TestSparkCmdTemplate extends AbstractCmdTemplate { protected StdOption userOp = option( null, - AppKeys.JOB_COMMON_SUBMIT_USER, + CliKeys.JOB_COMMON_SUBMIT_USER, new String[] {"-u", "--user"}, "specify user", true, @@ -67,7 +66,7 @@ public class TestSparkCmdTemplate extends AbstractCmdTemplate { private StdOption passwordOp = option( null, - AppKeys.JOB_COMMON_SUBMIT_PASSWORD, + CliKeys.JOB_COMMON_SUBMIT_PASSWORD, new String[] {"-pwd", "--passwd"}, "specify user password", true, @@ -95,7 +94,7 @@ public class TestSparkCmdTemplate extends AbstractCmdTemplate { private StdOption> confMap = option( null, - AppKeys.JOB_PARAM_CONF, + CliKeys.JOB_PARAM_CONF, new String[] {"-confMap"}, "confMap", true, diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/test/java/org/apache/linkis/cli/application/interactor/job/LinkisJobStatusTest.java b/linkis-computation-governance/linkis-client/linkis-cli/src/test/java/org/apache/linkis/cli/application/interactor/job/LinkisJobStatusTest.java similarity index 95% rename from linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/test/java/org/apache/linkis/cli/application/interactor/job/LinkisJobStatusTest.java rename to linkis-computation-governance/linkis-client/linkis-cli/src/test/java/org/apache/linkis/cli/application/interactor/job/LinkisJobStatusTest.java index 1b3b7c67ec..ff71dd0c43 100644 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/test/java/org/apache/linkis/cli/application/interactor/job/LinkisJobStatusTest.java +++ b/linkis-computation-governance/linkis-client/linkis-cli/src/test/java/org/apache/linkis/cli/application/interactor/job/LinkisJobStatusTest.java @@ -17,6 +17,8 @@ package org.apache.linkis.cli.application.interactor.job; +import org.apache.linkis.cli.application.interactor.job.common.LinkisJobStatus; + import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.DisplayName; import org.junit.jupiter.api.Test; diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/test/java/org/apache/linkis/cli/application/interactor/job/builder/ProcessKeyUtilsTest.java b/linkis-computation-governance/linkis-client/linkis-cli/src/test/java/org/apache/linkis/cli/application/interactor/job/builder/KeyParserTest.java similarity index 81% rename from linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/test/java/org/apache/linkis/cli/application/interactor/job/builder/ProcessKeyUtilsTest.java rename to linkis-computation-governance/linkis-client/linkis-cli/src/test/java/org/apache/linkis/cli/application/interactor/job/builder/KeyParserTest.java index cc579bec17..4c6c494c20 100644 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/test/java/org/apache/linkis/cli/application/interactor/job/builder/ProcessKeyUtilsTest.java +++ b/linkis-computation-governance/linkis-client/linkis-cli/src/test/java/org/apache/linkis/cli/application/interactor/job/builder/KeyParserTest.java @@ -17,7 +17,8 @@ package org.apache.linkis.cli.application.interactor.job.builder; -import org.apache.linkis.cli.application.constants.AppKeys; +import org.apache.linkis.cli.application.constants.CliKeys; +import org.apache.linkis.cli.application.interactor.job.common.KeyParser; import java.util.HashMap; import java.util.Map; @@ -26,17 +27,17 @@ import org.junit.jupiter.api.DisplayName; import org.junit.jupiter.api.Test; -public class ProcessKeyUtilsTest { +public class KeyParserTest { @Test @DisplayName("removePrefixForKeysInMapTest") public void removePrefixForKeysInMapTest() { Map map = new HashMap<>(); - map.put(AppKeys.JOB_PARAM_CONF, new Object()); + map.put(CliKeys.JOB_PARAM_CONF, new Object()); map.put("name", new Object()); - Map stringObjectMap = ProcessKeyUtils.removePrefixForKeysInMap(map); + Map stringObjectMap = KeyParser.removePrefixForKeysInMap(map); Assertions.assertTrue(1 == stringObjectMap.size()); } } diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/test/java/org/apache/linkis/cli/application/interactor/job/subtype/LinkisManSubTypeTest.java b/linkis-computation-governance/linkis-client/linkis-cli/src/test/java/org/apache/linkis/cli/application/interactor/job/subtype/JobCmdSubTypeTest.java similarity index 75% rename from linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/test/java/org/apache/linkis/cli/application/interactor/job/subtype/LinkisManSubTypeTest.java rename to linkis-computation-governance/linkis-client/linkis-cli/src/test/java/org/apache/linkis/cli/application/interactor/job/subtype/JobCmdSubTypeTest.java index 699eaaf8b0..c720e7f2c2 100644 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/test/java/org/apache/linkis/cli/application/interactor/job/subtype/LinkisManSubTypeTest.java +++ b/linkis-computation-governance/linkis-client/linkis-cli/src/test/java/org/apache/linkis/cli/application/interactor/job/subtype/JobCmdSubTypeTest.java @@ -17,22 +17,24 @@ package org.apache.linkis.cli.application.interactor.job.subtype; +import org.apache.linkis.cli.application.interactor.job.jobcmd.JobCmdSubType; + import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.DisplayName; import org.junit.jupiter.api.Test; -public class LinkisManSubTypeTest { +public class JobCmdSubTypeTest { @Test @DisplayName("enumTest") public void enumTest() { - String killName = LinkisManSubType.KILL.getName(); - String logName = LinkisManSubType.LOG.getName(); - String descName = LinkisManSubType.DESC.getName(); - String statusName = LinkisManSubType.STATUS.getName(); - String listName = LinkisManSubType.LIST.getName(); - String resultName = LinkisManSubType.RESULT.getName(); + String killName = JobCmdSubType.KILL.getName(); + String logName = JobCmdSubType.LOG.getName(); + String descName = JobCmdSubType.DESC.getName(); + String statusName = JobCmdSubType.STATUS.getName(); + String listName = JobCmdSubType.LIST.getName(); + String resultName = JobCmdSubType.RESULT.getName(); Assertions.assertEquals("kill", killName); Assertions.assertEquals("log", logName); diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/test/java/org/apache/linkis/cli/application/operator/once/OnceJobConstantsTest.java b/linkis-computation-governance/linkis-client/linkis-cli/src/test/java/org/apache/linkis/cli/application/operator/once/OnceJobConstantsTest.java similarity index 100% rename from linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/test/java/org/apache/linkis/cli/application/operator/once/OnceJobConstantsTest.java rename to linkis-computation-governance/linkis-client/linkis-cli/src/test/java/org/apache/linkis/cli/application/operator/once/OnceJobConstantsTest.java diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/test/java/org/apache/linkis/cli/application/operator/ujes/UJESConstantsTest.java b/linkis-computation-governance/linkis-client/linkis-cli/src/test/java/org/apache/linkis/cli/application/operator/ujes/UJESConstantsTest.java similarity index 100% rename from linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/test/java/org/apache/linkis/cli/application/operator/ujes/UJESConstantsTest.java rename to linkis-computation-governance/linkis-client/linkis-cli/src/test/java/org/apache/linkis/cli/application/operator/ujes/UJESConstantsTest.java diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/test/java/org/apache/linkis/cli/application/utils/UtilsTest.java b/linkis-computation-governance/linkis-client/linkis-cli/src/test/java/org/apache/linkis/cli/application/utils/UtilsTest.java similarity index 95% rename from linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/test/java/org/apache/linkis/cli/application/utils/UtilsTest.java rename to linkis-computation-governance/linkis-client/linkis-cli/src/test/java/org/apache/linkis/cli/application/utils/UtilsTest.java index 5af0e6ab85..8e07a5a6bb 100644 --- a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/test/java/org/apache/linkis/cli/application/utils/UtilsTest.java +++ b/linkis-computation-governance/linkis-client/linkis-cli/src/test/java/org/apache/linkis/cli/application/utils/UtilsTest.java @@ -28,7 +28,7 @@ public class UtilsTest { public void isValidExecIdTest() { String execId = "0001"; - boolean validExecId = Utils.isValidExecId(execId); + boolean validExecId = CliUtils.isValidExecId(execId); Assertions.assertTrue(validExecId); } } diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/test/resources/conf/linkis-cli.properties b/linkis-computation-governance/linkis-client/linkis-cli/src/test/resources/conf/linkis-cli.properties similarity index 100% rename from linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/test/resources/conf/linkis-cli.properties rename to linkis-computation-governance/linkis-client/linkis-cli/src/test/resources/conf/linkis-cli.properties diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/test/resources/conf/user.properties b/linkis-computation-governance/linkis-client/linkis-cli/src/test/resources/conf/user.properties similarity index 100% rename from linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/test/resources/conf/user.properties rename to linkis-computation-governance/linkis-client/linkis-cli/src/test/resources/conf/user.properties diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/test/resources/linkis-cli.properties b/linkis-computation-governance/linkis-client/linkis-cli/src/test/resources/linkis-cli.properties similarity index 100% rename from linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/test/resources/linkis-cli.properties rename to linkis-computation-governance/linkis-client/linkis-cli/src/test/resources/linkis-cli.properties diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/test/resources/log4j2.xml b/linkis-computation-governance/linkis-client/linkis-cli/src/test/resources/log4j2.xml similarity index 100% rename from linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/test/resources/log4j2.xml rename to linkis-computation-governance/linkis-client/linkis-cli/src/test/resources/log4j2.xml diff --git a/linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/test/resources/testScala.scala b/linkis-computation-governance/linkis-client/linkis-cli/src/test/resources/testScala.scala similarity index 100% rename from linkis-computation-governance/linkis-client/linkis-cli/linkis-cli-application/src/test/resources/testScala.scala rename to linkis-computation-governance/linkis-client/linkis-cli/src/test/resources/testScala.scala diff --git a/pom.xml b/pom.xml index fa691bc40c..a0e14f0fb4 100644 --- a/pom.xml +++ b/pom.xml @@ -1407,9 +1407,9 @@ org.apache.maven.plugins maven-surefire-plugin - --add-opens java.base/java.util=ALL-UNNAMED - --add-opens java.base/java.nio=ALL-UNNAMED - --add-opens java.base/java.lang=ALL-UNNAMED + + + From 0257e34ce58c47f8eab703da8e2d98e9570930df Mon Sep 17 00:00:00 2001 From: peacewong Date: Sat, 6 May 2023 16:22:56 +0800 Subject: [PATCH 093/261] Optimize jdbc driver --- .../config/ClientConfigBuilder.scala | 8 +- .../ujes/client/response/OpenLogResult.scala | 2 +- .../linkis/ujes/jdbc/UJESSQLDriver.java | 2 +- .../linkis/ujes/jdbc/utils/JDBCUtils.java | 2 - ...ection.scala => LinkisSQLConnection.scala} | 130 ++++++++++---- ...scala => LinkisSQLPreparedStatement.scala} | 4 +- ...atement.scala => LinkisSQLStatement.scala} | 166 ++++++++++-------- .../linkis/ujes/jdbc/UJESClientFactory.scala | 22 ++- .../ujes/jdbc/UJESSQLDatabaseMetaData.scala | 19 +- .../linkis/ujes/jdbc/UJESSQLDriverMain.scala | 8 +- .../linkis/ujes/jdbc/UJESSQLResultSet.scala | 28 +-- .../linkis/ujes/jdbc/UJESSQLTypeParser.scala | 11 +- .../hook/JDBCDriverPreExecutionHook.scala | 4 +- .../jdbc/hook/impl/NoLimitExecutionHook.scala | 6 +- .../hook/impl/TableauPreExecutionHook.scala | 5 +- .../linkis/ujes/jdbc/CreateConnection.java | 6 +- .../apache/linkis/ujes/jdbc/JDBCSpiTest.java | 12 +- ...tTest.java => LinkisSQLStatementTest.java} | 8 +- .../jdbc/UJESSQLDatabaseMetaDataTest.java | 2 +- .../jdbc/UJESSQLPreparedStatementTest.java | 4 +- .../ujes/jdbc/UJESSQLResultSetTest.java | 4 +- .../httpclient/dws/response/DWSResult.scala | 4 +- 22 files changed, 291 insertions(+), 166 deletions(-) rename linkis-computation-governance/linkis-jdbc-driver/src/main/scala/org/apache/linkis/ujes/jdbc/{UJESSQLConnection.scala => LinkisSQLConnection.scala} (73%) rename linkis-computation-governance/linkis-jdbc-driver/src/main/scala/org/apache/linkis/ujes/jdbc/{UJESSQLPreparedStatement.scala => LinkisSQLPreparedStatement.scala} (98%) rename linkis-computation-governance/linkis-jdbc-driver/src/main/scala/org/apache/linkis/ujes/jdbc/{UJESSQLStatement.scala => LinkisSQLStatement.scala} (72%) rename linkis-computation-governance/linkis-jdbc-driver/src/test/java/org/apache/linkis/ujes/jdbc/{UJESSQLStatementTest.java => LinkisSQLStatementTest.java} (95%) diff --git a/linkis-commons/linkis-httpclient/src/main/scala/org/apache/linkis/httpclient/config/ClientConfigBuilder.scala b/linkis-commons/linkis-httpclient/src/main/scala/org/apache/linkis/httpclient/config/ClientConfigBuilder.scala index c3e5afba30..b1fc579f3c 100644 --- a/linkis-commons/linkis-httpclient/src/main/scala/org/apache/linkis/httpclient/config/ClientConfigBuilder.scala +++ b/linkis-commons/linkis-httpclient/src/main/scala/org/apache/linkis/httpclient/config/ClientConfigBuilder.scala @@ -17,6 +17,7 @@ package org.apache.linkis.httpclient.config +import org.apache.linkis.common.exception.LinkisRetryException import org.apache.linkis.common.utils.{DefaultRetryHandler, RetryHandler} import org.apache.linkis.httpclient.authentication.AuthenticationStrategy import org.apache.linkis.httpclient.loadbalancer.LoadBalancerStrategy @@ -38,7 +39,12 @@ class ClientConfigBuilder protected () { protected var readTimeout: Long = _ protected var maxConnection: Int = _ protected var retryEnabled: Boolean = true - protected var retryHandler: RetryHandler = new DefaultRetryHandler + + protected var retryHandler: RetryHandler = { + val retryHandler = new DefaultRetryHandler + retryHandler.addRetryException(classOf[LinkisRetryException]) + retryHandler + } def addServerUrl(serverUrl: String): this.type = { this.serverUrl = serverUrl diff --git a/linkis-computation-governance/linkis-client/linkis-computation-client/src/main/scala/org/apache/linkis/ujes/client/response/OpenLogResult.scala b/linkis-computation-governance/linkis-client/linkis-computation-client/src/main/scala/org/apache/linkis/ujes/client/response/OpenLogResult.scala index fe107a32ca..2de5758aea 100644 --- a/linkis-computation-governance/linkis-client/linkis-computation-client/src/main/scala/org/apache/linkis/ujes/client/response/OpenLogResult.scala +++ b/linkis-computation-governance/linkis-client/linkis-computation-client/src/main/scala/org/apache/linkis/ujes/client/response/OpenLogResult.scala @@ -26,7 +26,7 @@ import scala.beans.BeanProperty class OpenLogResult extends DWSResult { /** - * log[0] - info log[1] - warn log[2] - error log[3] - all (info + warn + error) + * log[0] - error log[1] - warn log[2] - info log[3] - all (info + warn + error) */ @BeanProperty var log: Array[String] = _ diff --git a/linkis-computation-governance/linkis-jdbc-driver/src/main/java/org/apache/linkis/ujes/jdbc/UJESSQLDriver.java b/linkis-computation-governance/linkis-jdbc-driver/src/main/java/org/apache/linkis/ujes/jdbc/UJESSQLDriver.java index 90e1f73563..392f4dd097 100644 --- a/linkis-computation-governance/linkis-jdbc-driver/src/main/java/org/apache/linkis/ujes/jdbc/UJESSQLDriver.java +++ b/linkis-computation-governance/linkis-jdbc-driver/src/main/java/org/apache/linkis/ujes/jdbc/UJESSQLDriver.java @@ -58,7 +58,7 @@ public class UJESSQLDriver extends UJESSQLDriverMain implements Driver { static String ENABLE_DISCOVERY = "enableDiscovery"; static String ENABLE_LOADBALANCER = "enableLoadBalancer"; static String CREATOR = "creator"; - + static String TABLEAU = "tableau"; static String VARIABLE_HEADER = "var:"; static String PARAM_SPLIT = "&"; static String KV_SPLIT = "="; diff --git a/linkis-computation-governance/linkis-jdbc-driver/src/main/java/org/apache/linkis/ujes/jdbc/utils/JDBCUtils.java b/linkis-computation-governance/linkis-jdbc-driver/src/main/java/org/apache/linkis/ujes/jdbc/utils/JDBCUtils.java index 061acd42f9..8f7dcfed12 100644 --- a/linkis-computation-governance/linkis-jdbc-driver/src/main/java/org/apache/linkis/ujes/jdbc/utils/JDBCUtils.java +++ b/linkis-computation-governance/linkis-jdbc-driver/src/main/java/org/apache/linkis/ujes/jdbc/utils/JDBCUtils.java @@ -41,8 +41,6 @@ public static String convertPattern(final String pattern) { continue; } else if (c == '%') { result.append(".*"); - } else if (c == '_') { - result.append('.'); } else { result.append(Character.toLowerCase(c)); } diff --git a/linkis-computation-governance/linkis-jdbc-driver/src/main/scala/org/apache/linkis/ujes/jdbc/UJESSQLConnection.scala b/linkis-computation-governance/linkis-jdbc-driver/src/main/scala/org/apache/linkis/ujes/jdbc/LinkisSQLConnection.scala similarity index 73% rename from linkis-computation-governance/linkis-jdbc-driver/src/main/scala/org/apache/linkis/ujes/jdbc/UJESSQLConnection.scala rename to linkis-computation-governance/linkis-jdbc-driver/src/main/scala/org/apache/linkis/ujes/jdbc/LinkisSQLConnection.scala index 0d8403c274..9cd64f5281 100644 --- a/linkis-computation-governance/linkis-jdbc-driver/src/main/scala/org/apache/linkis/ujes/jdbc/UJESSQLConnection.scala +++ b/linkis-computation-governance/linkis-jdbc-driver/src/main/scala/org/apache/linkis/ujes/jdbc/LinkisSQLConnection.scala @@ -18,8 +18,12 @@ package org.apache.linkis.ujes.jdbc import org.apache.linkis.common.utils.{Logging, Utils} +import org.apache.linkis.manager.label.constant.LabelKeyConstant +import org.apache.linkis.manager.label.entity.engine.{EngineType, EngineTypeLabel, RunType} +import org.apache.linkis.manager.label.utils.EngineTypeLabelCreator import org.apache.linkis.ujes.client.UJESClient -import org.apache.linkis.ujes.client.request.JobExecuteAction.EngineType +import org.apache.linkis.ujes.client.request.JobSubmitAction +import org.apache.linkis.ujes.client.response.JobExecuteResult import org.apache.linkis.ujes.jdbc.UJESSQLDriverMain._ import org.apache.commons.lang3.StringUtils @@ -44,26 +48,41 @@ import java.sql.{ import java.util.Properties import java.util.concurrent.Executor -import scala.collection.{mutable, JavaConversions} +import scala.collection.JavaConverters._ -class UJESSQLConnection(private[jdbc] val ujesClient: UJESClient, props: Properties) +class LinkisSQLConnection(private[jdbc] val ujesClient: UJESClient, props: Properties) extends Connection with Logging { - private[jdbc] var creator = "IDE" + + private[jdbc] var creator = "JDBCDriver" + + private[jdbc] var tableau = false private[jdbc] val variableMap = { val params = props.getProperty(PARAMS) - val map = new mutable.HashMap[String, AnyRef] + val map = new util.HashMap[String, AnyRef] if (params != null) { params.split(PARAM_SPLIT).map(_.split(KV_SPLIT)).foreach { case Array(k, v) if k.startsWith(VARIABLE_HEADER) => - map += k.substring(VARIABLE_HEADER.length) -> v + map.put(k.substring(VARIABLE_HEADER.length), v) case Array(CREATOR, v) => creator = v case _ => } } - map.toMap + map + } + + def isTableau(): Boolean = { + val params = props.getProperty(PARAMS) + if (params != null) { + params.split(PARAM_SPLIT).map(_.split(KV_SPLIT)).foreach { + case Array(TABLEAU, v) => + tableau = true + case _ => + } + } + tableau } private[jdbc] val dbName = @@ -80,23 +99,21 @@ class UJESSQLConnection(private[jdbc] val ujesClient: UJESClient, props: Propert private[jdbc] val serverURL = props.getProperty("URL") - private val engineTypeMap: mutable.HashMap[String, EngineType] = new mutable.HashMap() + private val labelMap: util.Map[String, AnyRef] = new util.HashMap[String, AnyRef] - private[jdbc] def getEngineType: EngineType = { - if (engineTypeMap.isEmpty) { - engineTypeMap.put(EngineType.SPARK.toString, EngineType.SPARK) - engineTypeMap.put(EngineType.HIVE.toString, EngineType.HIVE) - engineTypeMap.put(EngineType.JDBC.toString, EngineType.JDBC) - engineTypeMap.put(EngineType.PYTHON.toString, EngineType.PYTHON) - engineTypeMap.put(EngineType.SHELL.toString, EngineType.SHELL) - engineTypeMap.put(EngineType.PRESTO.toString, EngineType.PRESTO) - } - val engineType: EngineType = EngineType.PRESTO + private val startupParams: util.Map[String, AnyRef] = new util.HashMap[String, AnyRef] + + private val runtimeParams: util.Map[String, AnyRef] = new util.HashMap[String, AnyRef] + + private[jdbc] def getEngineType: EngineTypeLabel = { + val engineType: EngineTypeLabel = + EngineTypeLabelCreator.createEngineTypeLabel(EngineType.TRINO.toString) if (props.containsKey(PARAMS)) { val params = props.getProperty(PARAMS) if (params != null & params.length() > 0) { params.split(PARAM_SPLIT).map(_.split(KV_SPLIT)).foreach { - case Array(k, v) if k.equals(UJESSQLDriver.ENGINE_TYPE) => return engineTypeMap(v) + case Array(k, v) if k.equals(UJESSQLDriver.ENGINE_TYPE) => + return EngineTypeLabelCreator.createEngineTypeLabel(v) case _ => } } @@ -121,28 +138,30 @@ class UJESSQLConnection(private[jdbc] val ujesClient: UJESClient, props: Propert def getProps: Properties = props - def removeStatement(statement: UJESSQLStatement): Unit = runningSQLStatements.remove(statement) + def removeStatement(statement: LinkisSQLStatement): Unit = runningSQLStatements.remove(statement) - override def createStatement(): Statement = createStatementAndAdd(new UJESSQLStatement(this)) + override def createStatement(): Statement = createStatementAndAdd(new LinkisSQLStatement(this)) - override def prepareStatement(sql: String): UJESSQLPreparedStatement = { - val statement = createStatementAndAdd(new UJESSQLPreparedStatement(this, sql)) + override def prepareStatement(sql: String): LinkisSQLPreparedStatement = { + val statement = createStatementAndAdd(new LinkisSQLPreparedStatement(this, sql)) statement.clearQuery() statement } override def createStatement(resultSetType: Int, resultSetConcurrency: Int): Statement = { - if (resultSetConcurrency != ResultSet.CONCUR_READ_ONLY) + if (resultSetConcurrency != ResultSet.CONCUR_READ_ONLY) { throw new SQLException( "Statement with resultset concurrency " + resultSetConcurrency + " is not supported", "HYC00" ) - if (resultSetType == ResultSet.TYPE_SCROLL_SENSITIVE) + } + if (resultSetType == ResultSet.TYPE_SCROLL_SENSITIVE) { throw new SQLException( "Statement with resultset type " + resultSetType + " is not supported", "HYC00" ) - createStatementAndAdd(new UJESSQLStatement(this)) + } + createStatementAndAdd(new LinkisSQLStatement(this)) } override def prepareStatement(sql: String, autoGeneratedKeys: Int): PreparedStatement = @@ -157,9 +176,7 @@ class UJESSQLConnection(private[jdbc] val ujesClient: UJESClient, props: Propert override def getMetaData: DatabaseMetaData = throwWhenClosed(new UJESSQLDatabaseMetaData(this)) override def close(): Unit = { - JavaConversions - .asScalaBuffer(runningSQLStatements) - .foreach(statement => Utils.tryQuietly(statement.close())) + runningSQLStatements.asScala.foreach(statement => Utils.tryQuietly(statement.close())) closed = true } @@ -305,15 +322,17 @@ class UJESSQLConnection(private[jdbc] val ujesClient: UJESClient, props: Propert throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_CONNECTION, "createStruct not supported") override def setSchema(schema: String): Unit = throwWhenClosed { - if (StringUtils.isBlank(schema)) + if (StringUtils.isBlank(schema)) { throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_STATEMENT, "schema is empty!") + } createStatement().execute("use " + schema) } override def getSchema: String = throwWhenClosed { val resultSet = createStatement().executeQuery("SELECT current_database()") - if (!resultSet.next()) + if (!resultSet.next()) { throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_STATEMENT, "Get schema failed!") + } resultSet.getString(1) } @@ -337,4 +356,53 @@ class UJESSQLConnection(private[jdbc] val ujesClient: UJESClient, props: Propert override def isWrapperFor(iface: Class[_]): Boolean = throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_CONNECTION, "isWrapperFor not supported") + def addLabels(labels: util.Map[String, AnyRef]): Unit = { + labelMap.putAll(labels) + } + + def addStartUpParams(params: util.Map[String, AnyRef]): Unit = { + startupParams.putAll(params) + } + + def addRuntimeParams(params: util.Map[String, AnyRef]): Unit = { + runtimeParams.putAll(params) + } + + def engineToCodeType(engine: String): String = { + val runType = EngineType.mapStringToEngineType(engine) match { + case EngineType.SPARK => RunType.SQL + case EngineType.HIVE => RunType.HIVE + case EngineType.TRINO => RunType.TRINO_SQL + case EngineType.PRESTO => RunType.PRESTO_SQL + case EngineType.ELASTICSEARCH => RunType.ES_SQL + case EngineType.JDBC => RunType.JDBC + case EngineType.PYTHON => RunType.SHELL + case _ => RunType.SQL + } + runType.toString + } + + private[jdbc] def toSubmit(code: String): JobExecuteResult = { + val engineTypeLabel = getEngineType + labelMap.put(LabelKeyConstant.ENGINE_TYPE_KEY, engineTypeLabel.getStringValue) + labelMap.put(LabelKeyConstant.USER_CREATOR_TYPE_KEY, s"$user-$creator") + labelMap.put(LabelKeyConstant.CODE_TYPE_KEY, engineToCodeType(engineTypeLabel.getEngineType)) + + val jobSubmitAction = JobSubmitAction.builder + .addExecuteCode(code) + .setStartupParams(startupParams) + .setUser(user) + .addExecuteUser(user) + .setLabels(labelMap) + .setRuntimeParams(runtimeParams) + .setVariableMap(variableMap) + .build + + val result = ujesClient.submit(jobSubmitAction) + if (result.getStatus != 0) { + throw new SQLException(result.getMessage) + } + result + } + } diff --git a/linkis-computation-governance/linkis-jdbc-driver/src/main/scala/org/apache/linkis/ujes/jdbc/UJESSQLPreparedStatement.scala b/linkis-computation-governance/linkis-jdbc-driver/src/main/scala/org/apache/linkis/ujes/jdbc/LinkisSQLPreparedStatement.scala similarity index 98% rename from linkis-computation-governance/linkis-jdbc-driver/src/main/scala/org/apache/linkis/ujes/jdbc/UJESSQLPreparedStatement.scala rename to linkis-computation-governance/linkis-jdbc-driver/src/main/scala/org/apache/linkis/ujes/jdbc/LinkisSQLPreparedStatement.scala index 6328da99e2..326a16bc7c 100644 --- a/linkis-computation-governance/linkis-jdbc-driver/src/main/scala/org/apache/linkis/ujes/jdbc/UJESSQLPreparedStatement.scala +++ b/linkis-computation-governance/linkis-jdbc-driver/src/main/scala/org/apache/linkis/ujes/jdbc/LinkisSQLPreparedStatement.scala @@ -36,8 +36,8 @@ import java.sql.{ import java.util import java.util.Calendar -class UJESSQLPreparedStatement(ujesSQLConnection: UJESSQLConnection, sql: String) - extends UJESSQLStatement(ujesSQLConnection) +class LinkisSQLPreparedStatement(ujesSQLConnection: LinkisSQLConnection, sql: String) + extends LinkisSQLStatement(ujesSQLConnection) with PreparedStatement { private val parameters = new util.HashMap[Int, Any] diff --git a/linkis-computation-governance/linkis-jdbc-driver/src/main/scala/org/apache/linkis/ujes/jdbc/UJESSQLStatement.scala b/linkis-computation-governance/linkis-jdbc-driver/src/main/scala/org/apache/linkis/ujes/jdbc/LinkisSQLStatement.scala similarity index 72% rename from linkis-computation-governance/linkis-jdbc-driver/src/main/scala/org/apache/linkis/ujes/jdbc/UJESSQLStatement.scala rename to linkis-computation-governance/linkis-jdbc-driver/src/main/scala/org/apache/linkis/ujes/jdbc/LinkisSQLStatement.scala index 916bdbaa93..1f6d5fc114 100644 --- a/linkis-computation-governance/linkis-jdbc-driver/src/main/scala/org/apache/linkis/ujes/jdbc/UJESSQLStatement.scala +++ b/linkis-computation-governance/linkis-jdbc-driver/src/main/scala/org/apache/linkis/ujes/jdbc/LinkisSQLStatement.scala @@ -19,19 +19,21 @@ package org.apache.linkis.ujes.jdbc import org.apache.linkis.common.exception.ErrorException import org.apache.linkis.common.utils.{Logging, Utils} -import org.apache.linkis.ujes.client.request.JobExecuteAction -import org.apache.linkis.ujes.client.request.JobExecuteAction.EngineType +import org.apache.linkis.governance.common.entity.ExecutionNodeStatus +import org.apache.linkis.ujes.client.request.OpenLogAction import org.apache.linkis.ujes.client.response.JobExecuteResult import org.apache.linkis.ujes.jdbc.hook.JDBCDriverPreExecutionHook +import org.apache.commons.lang3.StringUtils + import java.sql.{Connection, ResultSet, SQLWarning, Statement} +import java.util import java.util.concurrent.TimeUnit -import scala.collection.JavaConverters.mapAsJavaMapConverter import scala.concurrent.TimeoutException import scala.concurrent.duration.Duration -class UJESSQLStatement(private[jdbc] val ujesSQLConnection: UJESSQLConnection) +class LinkisSQLStatement(private[jdbc] val ujesSQLConnection: LinkisSQLConnection) extends Statement with Logging { @@ -42,8 +44,13 @@ class UJESSQLStatement(private[jdbc] val ujesSQLConnection: UJESSQLConnection) private var fetchSize = 100 private var queryTimeout = 0 + private var logPath: String = null + private var queryEnd = false + private var logFromLen = 0 + private val logSize = 100 + private[jdbc] def throwWhenClosed[T](op: => T): T = ujesSQLConnection.throwWhenClosed { if (isClosed) throw new UJESSQLException(UJESSQLErrorCode.STATEMENT_CLOSED) else op @@ -89,11 +96,12 @@ class UJESSQLStatement(private[jdbc] val ujesSQLConnection: UJESSQLConnection) override def setMaxRows(max: Int): Unit = this.maxRows = max - override def setEscapeProcessing(enable: Boolean): Unit = if (enable) + override def setEscapeProcessing(enable: Boolean): Unit = if (enable) { throw new UJESSQLException( UJESSQLErrorCode.NOSUPPORT_STATEMENT, "setEscapeProcessing not supported" ) + } override def getQueryTimeout: Int = queryTimeout @@ -108,88 +116,62 @@ class UJESSQLStatement(private[jdbc] val ujesSQLConnection: UJESSQLConnection) override def setCursorName(name: String): Unit = throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_STATEMENT, "setCursorName not supported") - override def execute(sql: String): Boolean = Utils.tryCatch(throwWhenClosed { + override def execute(sql: String): Boolean = throwWhenClosed { var parsedSQL = sql JDBCDriverPreExecutionHook.getPreExecutionHooks.foreach { preExecution => - parsedSQL = preExecution.callPreExecutionHook(parsedSQL) + parsedSQL = preExecution.callPreExecutionHook(parsedSQL, !ujesSQLConnection.isTableau()) } logger.info(s"begin to execute sql ${parsedSQL}") - val action = JobExecuteAction - .builder() - .setEngineType(ujesSQLConnection.getEngineType) - .addExecuteCode(parsedSQL) - .setCreator(ujesSQLConnection.creator) - .setUser(ujesSQLConnection.user) - if (ujesSQLConnection.variableMap.nonEmpty) { - action.setVariableMap(ujesSQLConnection.variableMap.asJava) - } - jobExecuteResult = - Utils.tryCatch(ujesSQLConnection.ujesClient.execute(action.build())) { t: Throwable => - logger.error("UJESClient failed to get result", t) - null - } - // jobExecuteResult = ujesSQLConnection.ujesClient.execute(action.build()) queryEnd = false - var status = ujesSQLConnection.ujesClient.status(jobExecuteResult) - val atMost = - if (queryTimeout > 0) Duration(queryTimeout, TimeUnit.MILLISECONDS) else Duration.Inf - if (!status.isCompleted) Utils.tryThrow { - Utils.waitUntil( - () => { - status = ujesSQLConnection.ujesClient.status(jobExecuteResult) - status.isCompleted || closed - }, - atMost, - 100, - 10000 - ) - } { - case t: TimeoutException => - if (queryTimeout > 0) clearQuery() - new UJESSQLException(UJESSQLErrorCode.QUERY_TIMEOUT, "query has been timeout!").initCause(t) - case t => t - } - if (!closed) { + logPath = null + Utils.tryFinally { + jobExecuteResult = ujesSQLConnection.toSubmit(parsedSQL) + val atMost = + if (queryTimeout > 0) Duration(queryTimeout, TimeUnit.MILLISECONDS) else Duration.Inf var jobInfo = ujesSQLConnection.ujesClient.getJobInfo(jobExecuteResult) - if (status.isFailed) + logPath = jobInfo.getRequestPersistTask.getLogPath + if (!ExecutionNodeStatus.isCompleted(ExecutionNodeStatus.valueOf(jobInfo.getJobStatus))) { + Utils.tryThrow { + Utils.waitUntil( + () => { + jobInfo = ujesSQLConnection.ujesClient.getJobInfo(jobExecuteResult) + ExecutionNodeStatus.isCompleted( + ExecutionNodeStatus.valueOf(jobInfo.getJobStatus) + ) || closed + }, + atMost, + 100, + 10000 + ) + } { + case t: TimeoutException => + if (queryTimeout > 0) clearQuery() + logPath = jobInfo.getRequestPersistTask.getLogPath + new UJESSQLException(UJESSQLErrorCode.QUERY_TIMEOUT, "query has been timeout!") + .initCause(t) + case t => t + } + } + logPath = jobInfo.getRequestPersistTask.getLogPath + if (!ExecutionNodeStatus.isSucceed(ExecutionNodeStatus.valueOf(jobInfo.getJobStatus))) { throw new ErrorException( jobInfo.getRequestPersistTask.getErrCode, jobInfo.getRequestPersistTask.getErrDesc ) - val jobInfoStatus = jobInfo.getJobStatus - if (!jobInfoStatus.equals("Succeed")) Utils.tryThrow { - Utils.waitUntil( - () => { - jobInfo = ujesSQLConnection.ujesClient.getJobInfo(jobExecuteResult) - val state = jobInfo.getJobStatus match { - case "Failed" | "Cancelled" | "Timeout" | "Succeed" => true - case _ => false - } - state || closed - }, - atMost, - 100, - 10000 - ) - } { - case t: TimeoutException => - if (queryTimeout > 0) clearQuery() - new UJESSQLException(UJESSQLErrorCode.QUERY_TIMEOUT, "query has been timeout!") - .initCause(t) - case t => t } + logger.info(s"end to execute sql ${parsedSQL}") val resultSetList = jobInfo.getResultSetList(ujesSQLConnection.ujesClient) logger.info(s"resultSetList is ${resultSetList.mkString(",")}") - queryEnd = true - if (resultSetList != null) { + if (resultSetList != null && resultSetList.nonEmpty) { resultSet = new UJESSQLResultSet(resultSetList, this, maxRows, fetchSize) true - } else false - } else throw new UJESSQLException(UJESSQLErrorCode.STATEMENT_CLOSED, "Statement is closed.") - }) { case t: Throwable => - logger.error("任务执行失败", t) - false + } else { + false + } + } { + queryEnd = true + } } def getJobExcuteResult: JobExecuteResult = jobExecuteResult @@ -201,13 +183,12 @@ class UJESSQLStatement(private[jdbc] val ujesSQLConnection: UJESSQLConnection) override def getMoreResults: Boolean = false override def setFetchDirection(direction: Int): Unit = - throwWhenClosed( - if (direction != ResultSet.FETCH_FORWARD) - throw new UJESSQLException( - UJESSQLErrorCode.NOSUPPORT_STATEMENT, - "only FETCH_FORWARD is supported." - ) - ) + throwWhenClosed(if (direction != ResultSet.FETCH_FORWARD) { + throw new UJESSQLException( + UJESSQLErrorCode.NOSUPPORT_STATEMENT, + "only FETCH_FORWARD is supported." + ) + }) override def getFetchDirection: Int = throwWhenClosed(ResultSet.FETCH_FORWARD) @@ -299,4 +280,33 @@ class UJESSQLStatement(private[jdbc] val ujesSQLConnection: UJESSQLConnection) throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_STATEMENT, "unwrap not supported") override def isWrapperFor(iface: Class[_]): Boolean = false + + /** + * log[0] error log[1] warn log[2] info log[3] all (info + warn + error) + * @return + */ + def getAllLog(): Array[String] = { + if (queryEnd && StringUtils.isNotBlank(logPath)) { + val openLogAction = + OpenLogAction.newBuilder().setLogPath(logPath).setProxyUser(ujesSQLConnection.user).build() + ujesSQLConnection.ujesClient.openLog(openLogAction).getLog + } else { + Array.empty[String] + } + } + + /** + * log[0] error log[1] warn log[2] info log[3] all (info + warn + error) + * @return + */ + def getIncrementalLog(): util.List[String] = { + if (null != jobExecuteResult && !queryEnd) { + val logObj = ujesSQLConnection.ujesClient.log(jobExecuteResult, logFromLen, logSize) + logFromLen = logObj.fromLine + logObj.getLog + } else { + new util.ArrayList[String] + } + } + } diff --git a/linkis-computation-governance/linkis-jdbc-driver/src/main/scala/org/apache/linkis/ujes/jdbc/UJESClientFactory.scala b/linkis-computation-governance/linkis-jdbc-driver/src/main/scala/org/apache/linkis/ujes/jdbc/UJESClientFactory.scala index 60f4d88af5..517f8b07ae 100644 --- a/linkis-computation-governance/linkis-jdbc-driver/src/main/scala/org/apache/linkis/ujes/jdbc/UJESClientFactory.scala +++ b/linkis-computation-governance/linkis-jdbc-driver/src/main/scala/org/apache/linkis/ujes/jdbc/UJESClientFactory.scala @@ -35,15 +35,27 @@ object UJESClientFactory extends Logging { def getUJESClient(props: Properties): UJESClient = { val host = props.getProperty(HOST) val port = props.getProperty(PORT) + val user = props.getProperty(USER) val serverUrl = if (StringUtils.isNotBlank(port)) s"http://$host:$port" else "http://" + host - if (ujesClients.containsKey(serverUrl)) ujesClients.get(serverUrl) - else - serverUrl.intern synchronized { - if (ujesClients.containsKey(serverUrl)) return ujesClients.get(serverUrl) + val uniqueKey = s"${serverUrl}_$user" + if (ujesClients.containsKey(uniqueKey)) { + logger.info("Clients with the same JDBC unique key({}) will get it directly", uniqueKey) + ujesClients.get(uniqueKey) + } else { + uniqueKey.intern synchronized { + if (ujesClients.containsKey(uniqueKey)) { + logger.info("Clients with the same JDBC unique key({}) will get it directly", uniqueKey) + return ujesClients.get(uniqueKey) + } + logger.info( + "The same Client does not exist for the JDBC unique key({}), a new Client will be created", + uniqueKey + ) val ujesClient = createUJESClient(serverUrl, props) - ujesClients.put(serverUrl, ujesClient) + ujesClients.put(uniqueKey, ujesClient) ujesClient } + } } private def createUJESClient(serverUrl: String, props: Properties): UJESClient = { diff --git a/linkis-computation-governance/linkis-jdbc-driver/src/main/scala/org/apache/linkis/ujes/jdbc/UJESSQLDatabaseMetaData.scala b/linkis-computation-governance/linkis-jdbc-driver/src/main/scala/org/apache/linkis/ujes/jdbc/UJESSQLDatabaseMetaData.scala index f2f0b9a106..ad856e81e0 100644 --- a/linkis-computation-governance/linkis-jdbc-driver/src/main/scala/org/apache/linkis/ujes/jdbc/UJESSQLDatabaseMetaData.scala +++ b/linkis-computation-governance/linkis-jdbc-driver/src/main/scala/org/apache/linkis/ujes/jdbc/UJESSQLDatabaseMetaData.scala @@ -29,7 +29,7 @@ import java.util import scala.collection.JavaConversions._ -class UJESSQLDatabaseMetaData(ujesSQLConnection: UJESSQLConnection) +class UJESSQLDatabaseMetaData(ujesSQLConnection: LinkisSQLConnection) extends DatabaseMetaData with Logging { override def allProceduresAreCallable(): Boolean = false @@ -358,7 +358,9 @@ class UJESSQLDatabaseMetaData(ujesSQLConnection: UJESSQLConnection) tableNamePattern: String, types: Array[String] ): ResultSet = { - val resultCatalog = if (StringUtils.isNotBlank(catalog)) { + val resultCatalog = if (StringUtils.isNotBlank(schemaPattern)) { + schemaPattern + } else if (StringUtils.isNotBlank(catalog)) { catalog } else { s"${getUserName}_ind" @@ -374,11 +376,16 @@ class UJESSQLDatabaseMetaData(ujesSQLConnection: UJESSQLConnection) if (table.get("isView").asInstanceOf[Boolean]) TableType.VIEW.name() else TableType.TABLE.name() val resultTable = new util.HashMap[String, String]() + val tableName = table.get("tableName").asInstanceOf[String] resultTable.put("catalog", resultCatalog) - resultTable.put("tableName", table.get("tableName").asInstanceOf[String]) + resultTable.put("tableName", tableName) resultTable.put("tableType", tableType) if (null == types || types.contains(tableType)) { - resultTables.add(resultTable) + if ( + StringUtils.isNotBlank(tableNamePattern) && tableNamePattern.equalsIgnoreCase(tableName) + ) { + resultTables.add(resultTable) + } } } val resultSet: LinkisMetaDataResultSet[util.Map[String, String]] = @@ -471,7 +478,9 @@ class UJESSQLDatabaseMetaData(ujesSQLConnection: UJESSQLConnection) tableNamePattern: String, columnNamePattern: String ): ResultSet = { - val resultCatalog = if (StringUtils.isNotBlank(catalog)) { + val resultCatalog = if (StringUtils.isNotBlank(schemaPattern)) { + schemaPattern + } else if (StringUtils.isNotBlank(catalog)) { catalog } else { s"${getUserName}_ind" diff --git a/linkis-computation-governance/linkis-jdbc-driver/src/main/scala/org/apache/linkis/ujes/jdbc/UJESSQLDriverMain.scala b/linkis-computation-governance/linkis-jdbc-driver/src/main/scala/org/apache/linkis/ujes/jdbc/UJESSQLDriverMain.scala index 783713cf40..9fb4c0f605 100644 --- a/linkis-computation-governance/linkis-jdbc-driver/src/main/scala/org/apache/linkis/ujes/jdbc/UJESSQLDriverMain.scala +++ b/linkis-computation-governance/linkis-jdbc-driver/src/main/scala/org/apache/linkis/ujes/jdbc/UJESSQLDriverMain.scala @@ -41,8 +41,10 @@ class UJESSQLDriverMain extends Driver with Logging { props.putAll(parseURL(url)) logger.info(s"input url:$url, properties:$properties") val ujesClient = UJESClientFactory.getUJESClient(props) - new UJESSQLConnection(ujesClient, props) - } else throw new UJESSQLException(UJESSQLErrorCode.BAD_URL, "bad url: " + url) + new LinkisSQLConnection(ujesClient, props) + } else { + null + } override def acceptsURL(url: String): Boolean = url.startsWith(URL_PREFIX) @@ -144,6 +146,8 @@ object UJESSQLDriverMain { val ENABLE_LOADBALANCER = UJESSQLDriver.ENABLE_LOADBALANCER val CREATOR = UJESSQLDriver.CREATOR + val TABLEAU = UJESSQLDriver.TABLEAU + val VARIABLE_HEADER = UJESSQLDriver.VARIABLE_HEADER def getConnectionParams( diff --git a/linkis-computation-governance/linkis-jdbc-driver/src/main/scala/org/apache/linkis/ujes/jdbc/UJESSQLResultSet.scala b/linkis-computation-governance/linkis-jdbc-driver/src/main/scala/org/apache/linkis/ujes/jdbc/UJESSQLResultSet.scala index b8cf1b23b1..97bfc576e3 100644 --- a/linkis-computation-governance/linkis-jdbc-driver/src/main/scala/org/apache/linkis/ujes/jdbc/UJESSQLResultSet.scala +++ b/linkis-computation-governance/linkis-jdbc-driver/src/main/scala/org/apache/linkis/ujes/jdbc/UJESSQLResultSet.scala @@ -42,7 +42,7 @@ import java.sql.{ Time, Timestamp } -import java.util.Calendar +import java.util.{Calendar, Locale} import org.joda.time.DateTimeZone import org.joda.time.format.{ @@ -54,7 +54,7 @@ import org.joda.time.format.{ class UJESSQLResultSet( resultSetList: Array[String], - ujesStatement: UJESSQLStatement, + ujesStatement: LinkisSQLStatement, maxRows: Int, fetchSize: Int ) extends ResultSet @@ -75,10 +75,10 @@ class UJESSQLResultSet( private val pageSize: Int = 5000 private var path: String = _ private var metaData: util.List[util.Map[String, String]] = _ - private val statement: UJESSQLStatement = ujesStatement + private val statement: LinkisSQLStatement = ujesStatement - private val connection: UJESSQLConnection = - ujesStatement.getConnection.asInstanceOf[UJESSQLConnection] + private val connection: LinkisSQLConnection = + ujesStatement.getConnection.asInstanceOf[LinkisSQLConnection] private var valueWasNull: Boolean = false private var warningChain: SQLWarning = _ @@ -96,6 +96,10 @@ class UJESSQLResultSet( .toFormatter .withOffsetParsed + private val STRING_TYPE = "string" + + private val NULL_VALUE = "NULL" + private def getResultSetPath(resultSetList: Array[String]): String = { if (resultSetList.length > 0) { resultSetList(resultSetList.length - 1) @@ -229,10 +233,14 @@ class UJESSQLResultSet( } private def evaluate(dataType: String, value: String): Any = { + if (value == null || value.equals("null") || value.equals("NULL") || value.equals("Null")) { - value + dataType.toLowerCase(Locale.getDefault) match { + case "string" | "char" | "varchar" | "nvarchar" => value + case _ => null + } } else { - dataType.toLowerCase match { + dataType.toLowerCase(Locale.getDefault) match { case null => throw new UJESSQLException(UJESSQLErrorCode.METADATA_EMPTY) case "string" => value.toString case "short" => value.toShort @@ -518,11 +526,7 @@ class UJESSQLResultSet( override def getObject(columnIndex: Int): Object = { val any = getColumnValue(columnIndex) - if (wasNull()) { - throw new UJESSQLException(UJESSQLErrorCode.RESULTSET_ROWERROR, "Type is null") - } else { - any.asInstanceOf[Object] - } + any.asInstanceOf[Object] } override def getObject(columnLabel: String): Object = { diff --git a/linkis-computation-governance/linkis-jdbc-driver/src/main/scala/org/apache/linkis/ujes/jdbc/UJESSQLTypeParser.scala b/linkis-computation-governance/linkis-jdbc-driver/src/main/scala/org/apache/linkis/ujes/jdbc/UJESSQLTypeParser.scala index ba4cd3878f..541064b2d5 100644 --- a/linkis-computation-governance/linkis-jdbc-driver/src/main/scala/org/apache/linkis/ujes/jdbc/UJESSQLTypeParser.scala +++ b/linkis-computation-governance/linkis-jdbc-driver/src/main/scala/org/apache/linkis/ujes/jdbc/UJESSQLTypeParser.scala @@ -18,11 +18,13 @@ package org.apache.linkis.ujes.jdbc import java.sql.{SQLException, Timestamp, Types} +import java.util.Locale object UJESSQLTypeParser { def parserFromName(typeName: String): Int = { - typeName.toLowerCase match { + val typeNameLowerCase = typeName.toLowerCase(Locale.getDefault()) + typeName.toLowerCase() match { case null => throw new UJESSQLException(UJESSQLErrorCode.METADATA_EMPTY) case "string" => Types.NVARCHAR case "short" => Types.SMALLINT @@ -43,7 +45,12 @@ object UJESSQLTypeParser { case "bigint" => Types.BIGINT case "array" => Types.ARRAY case "map" => Types.JAVA_OBJECT - case _ => throw new SQLException(s"parameter type error,Type:$typeName") + case _ => + if (typeNameLowerCase.startsWith("decimal")) { + Types.DECIMAL + } else { + Types.NVARCHAR + } } } diff --git a/linkis-computation-governance/linkis-jdbc-driver/src/main/scala/org/apache/linkis/ujes/jdbc/hook/JDBCDriverPreExecutionHook.scala b/linkis-computation-governance/linkis-jdbc-driver/src/main/scala/org/apache/linkis/ujes/jdbc/hook/JDBCDriverPreExecutionHook.scala index 038ff38712..ed8936cc77 100644 --- a/linkis-computation-governance/linkis-jdbc-driver/src/main/scala/org/apache/linkis/ujes/jdbc/hook/JDBCDriverPreExecutionHook.scala +++ b/linkis-computation-governance/linkis-jdbc-driver/src/main/scala/org/apache/linkis/ujes/jdbc/hook/JDBCDriverPreExecutionHook.scala @@ -24,7 +24,7 @@ import scala.collection.mutable.ArrayBuffer trait JDBCDriverPreExecutionHook { - def callPreExecutionHook(sql: String): String + def callPreExecutionHook(sql: String, skip: Boolean): String } @@ -51,5 +51,5 @@ object JDBCDriverPreExecutionHook extends Logging { hooks.toArray } - def getPreExecutionHooks = preExecutionHooks + def getPreExecutionHooks: Array[JDBCDriverPreExecutionHook] = preExecutionHooks } diff --git a/linkis-computation-governance/linkis-jdbc-driver/src/main/scala/org/apache/linkis/ujes/jdbc/hook/impl/NoLimitExecutionHook.scala b/linkis-computation-governance/linkis-jdbc-driver/src/main/scala/org/apache/linkis/ujes/jdbc/hook/impl/NoLimitExecutionHook.scala index 18f94c0fde..103a544772 100644 --- a/linkis-computation-governance/linkis-jdbc-driver/src/main/scala/org/apache/linkis/ujes/jdbc/hook/impl/NoLimitExecutionHook.scala +++ b/linkis-computation-governance/linkis-jdbc-driver/src/main/scala/org/apache/linkis/ujes/jdbc/hook/impl/NoLimitExecutionHook.scala @@ -20,10 +20,12 @@ package org.apache.linkis.ujes.jdbc.hook.impl import org.apache.linkis.ujes.jdbc.UJESSQLDriverMain import org.apache.linkis.ujes.jdbc.hook.JDBCDriverPreExecutionHook +import java.util.Locale + class NoLimitExecutionHook extends JDBCDriverPreExecutionHook { - override def callPreExecutionHook(sql: String): String = { - if (UJESSQLDriverMain.LIMIT_ENABLED.toLowerCase.equals("false")) { + override def callPreExecutionHook(sql: String, skip: Boolean): String = { + if (UJESSQLDriverMain.LIMIT_ENABLED.equalsIgnoreCase("false")) { var noLimitSql = "--set ide.engine.no.limit.allow=true\n" + sql val lowerCaseLimitSql = noLimitSql.toLowerCase() if (lowerCaseLimitSql.contains("limit ") && lowerCaseLimitSql.contains("tableausql")) { diff --git a/linkis-computation-governance/linkis-jdbc-driver/src/main/scala/org/apache/linkis/ujes/jdbc/hook/impl/TableauPreExecutionHook.scala b/linkis-computation-governance/linkis-jdbc-driver/src/main/scala/org/apache/linkis/ujes/jdbc/hook/impl/TableauPreExecutionHook.scala index 63c4f53c9f..ec177b4de7 100644 --- a/linkis-computation-governance/linkis-jdbc-driver/src/main/scala/org/apache/linkis/ujes/jdbc/hook/impl/TableauPreExecutionHook.scala +++ b/linkis-computation-governance/linkis-jdbc-driver/src/main/scala/org/apache/linkis/ujes/jdbc/hook/impl/TableauPreExecutionHook.scala @@ -21,7 +21,10 @@ import org.apache.linkis.ujes.jdbc.hook.JDBCDriverPreExecutionHook class TableauPreExecutionHook extends JDBCDriverPreExecutionHook { - override def callPreExecutionHook(sql: String): String = { + override def callPreExecutionHook(sql: String, skip: Boolean): String = { + if (skip) { + return sql + } if ( sql.contains("CREATE INDEX") || sql .contains("CREATE TABLE") || sql.contains("INSERT INTO") || sql.contains("DROP TABLE") diff --git a/linkis-computation-governance/linkis-jdbc-driver/src/test/java/org/apache/linkis/ujes/jdbc/CreateConnection.java b/linkis-computation-governance/linkis-jdbc-driver/src/test/java/org/apache/linkis/ujes/jdbc/CreateConnection.java index 51d9e9953a..ca2c1e35fd 100644 --- a/linkis-computation-governance/linkis-jdbc-driver/src/test/java/org/apache/linkis/ujes/jdbc/CreateConnection.java +++ b/linkis-computation-governance/linkis-jdbc-driver/src/test/java/org/apache/linkis/ujes/jdbc/CreateConnection.java @@ -27,12 +27,12 @@ public class CreateConnection { - private static UJESSQLConnection conn; + private static LinkisSQLConnection conn; - public static UJESSQLConnection getConnection() throws ClassNotFoundException, SQLException { + public static LinkisSQLConnection getConnection() throws ClassNotFoundException, SQLException { Class.forName("org.apache.linkis.ujes.jdbc.UJESSQLDriver"); conn = - (UJESSQLConnection) + (LinkisSQLConnection) DriverManager.getConnection("jdbc:linkis://hostname:port", "username", "password"); return conn; } diff --git a/linkis-computation-governance/linkis-jdbc-driver/src/test/java/org/apache/linkis/ujes/jdbc/JDBCSpiTest.java b/linkis-computation-governance/linkis-jdbc-driver/src/test/java/org/apache/linkis/ujes/jdbc/JDBCSpiTest.java index 427a6b5fc4..3e76bb2856 100644 --- a/linkis-computation-governance/linkis-jdbc-driver/src/test/java/org/apache/linkis/ujes/jdbc/JDBCSpiTest.java +++ b/linkis-computation-governance/linkis-jdbc-driver/src/test/java/org/apache/linkis/ujes/jdbc/JDBCSpiTest.java @@ -29,21 +29,21 @@ * */ public class JDBCSpiTest { - private static UJESSQLConnection conn; + private static LinkisSQLConnection conn; - public static UJESSQLConnection getConnection() throws ClassNotFoundException, SQLException { + public static LinkisSQLConnection getConnection() throws ClassNotFoundException, SQLException { Class.forName("org.apache.linkis.ujes.jdbc.UJESSQLDriver"); conn = - (UJESSQLConnection) - DriverManager.getConnection("jdbc:linkis://hostname:port", "root", "123456"); + (LinkisSQLConnection) + DriverManager.getConnection("jdbc:linkis://127.0.0.1:9001", "root", "123456"); return conn; } @Test public void spiTest() { try { - UJESSQLConnection conn = - (UJESSQLConnection) + LinkisSQLConnection conn = + (LinkisSQLConnection) DriverManager.getConnection("jdbc:linkis://hostname:port", "username", "password"); Assertions.assertNotNull(conn); } catch (SQLException e) { diff --git a/linkis-computation-governance/linkis-jdbc-driver/src/test/java/org/apache/linkis/ujes/jdbc/UJESSQLStatementTest.java b/linkis-computation-governance/linkis-jdbc-driver/src/test/java/org/apache/linkis/ujes/jdbc/LinkisSQLStatementTest.java similarity index 95% rename from linkis-computation-governance/linkis-jdbc-driver/src/test/java/org/apache/linkis/ujes/jdbc/UJESSQLStatementTest.java rename to linkis-computation-governance/linkis-jdbc-driver/src/test/java/org/apache/linkis/ujes/jdbc/LinkisSQLStatementTest.java index e674dd5b23..3ebd21ae70 100644 --- a/linkis-computation-governance/linkis-jdbc-driver/src/test/java/org/apache/linkis/ujes/jdbc/UJESSQLStatementTest.java +++ b/linkis-computation-governance/linkis-jdbc-driver/src/test/java/org/apache/linkis/ujes/jdbc/LinkisSQLStatementTest.java @@ -33,9 +33,9 @@ * if you want to test this module,you must rewrite default parameters and SQL we used for local test * */ -public class UJESSQLStatementTest { - private static UJESSQLConnection conn; - private static UJESSQLStatement statement; +public class LinkisSQLStatementTest { + private static LinkisSQLConnection conn; + private static LinkisSQLStatement statement; private static int maxRows; private static int queryTimeout; private static String sql; @@ -48,7 +48,7 @@ public class UJESSQLStatementTest { public static void createConnection() { try { conn = CreateConnection.getConnection(); - statement = (UJESSQLStatement) conn.createStatement(); + statement = (LinkisSQLStatement) conn.createStatement(); } catch (ClassNotFoundException e) { e.printStackTrace(); } catch (SQLException e) { diff --git a/linkis-computation-governance/linkis-jdbc-driver/src/test/java/org/apache/linkis/ujes/jdbc/UJESSQLDatabaseMetaDataTest.java b/linkis-computation-governance/linkis-jdbc-driver/src/test/java/org/apache/linkis/ujes/jdbc/UJESSQLDatabaseMetaDataTest.java index 0dab63b3ff..e7c6ade996 100644 --- a/linkis-computation-governance/linkis-jdbc-driver/src/test/java/org/apache/linkis/ujes/jdbc/UJESSQLDatabaseMetaDataTest.java +++ b/linkis-computation-governance/linkis-jdbc-driver/src/test/java/org/apache/linkis/ujes/jdbc/UJESSQLDatabaseMetaDataTest.java @@ -34,7 +34,7 @@ import static org.junit.jupiter.api.Assertions.assertTrue; public class UJESSQLDatabaseMetaDataTest { - private static UJESSQLConnection conn; + private static LinkisSQLConnection conn; private static UJESSQLDatabaseMetaData dbmd; @BeforeAll diff --git a/linkis-computation-governance/linkis-jdbc-driver/src/test/java/org/apache/linkis/ujes/jdbc/UJESSQLPreparedStatementTest.java b/linkis-computation-governance/linkis-jdbc-driver/src/test/java/org/apache/linkis/ujes/jdbc/UJESSQLPreparedStatementTest.java index 54bc3d4538..da431c82e7 100644 --- a/linkis-computation-governance/linkis-jdbc-driver/src/test/java/org/apache/linkis/ujes/jdbc/UJESSQLPreparedStatementTest.java +++ b/linkis-computation-governance/linkis-jdbc-driver/src/test/java/org/apache/linkis/ujes/jdbc/UJESSQLPreparedStatementTest.java @@ -35,8 +35,8 @@ * */ public class UJESSQLPreparedStatementTest { - private static UJESSQLConnection conn; - private UJESSQLPreparedStatement preStatement; + private static LinkisSQLConnection conn; + private LinkisSQLPreparedStatement preStatement; @BeforeAll public static void getConnection() { diff --git a/linkis-computation-governance/linkis-jdbc-driver/src/test/java/org/apache/linkis/ujes/jdbc/UJESSQLResultSetTest.java b/linkis-computation-governance/linkis-jdbc-driver/src/test/java/org/apache/linkis/ujes/jdbc/UJESSQLResultSetTest.java index 0a545d8012..a8f0a179d0 100644 --- a/linkis-computation-governance/linkis-jdbc-driver/src/test/java/org/apache/linkis/ujes/jdbc/UJESSQLResultSetTest.java +++ b/linkis-computation-governance/linkis-jdbc-driver/src/test/java/org/apache/linkis/ujes/jdbc/UJESSQLResultSetTest.java @@ -33,8 +33,8 @@ public class UJESSQLResultSetTest { - private static UJESSQLConnection conn; - private UJESSQLPreparedStatement preStatement; + private static LinkisSQLConnection conn; + private LinkisSQLPreparedStatement preStatement; private UJESSQLResultSet resultSet; private UJESSQLResultSetMetaData metaData; diff --git a/linkis-spring-cloud-services/linkis-service-gateway/linkis-gateway-httpclient-support/src/main/scala/org/apache/linkis/httpclient/dws/response/DWSResult.scala b/linkis-spring-cloud-services/linkis-service-gateway/linkis-gateway-httpclient-support/src/main/scala/org/apache/linkis/httpclient/dws/response/DWSResult.scala index bada2315cb..3f00f61f5f 100644 --- a/linkis-spring-cloud-services/linkis-service-gateway/linkis-gateway-httpclient-support/src/main/scala/org/apache/linkis/httpclient/dws/response/DWSResult.scala +++ b/linkis-spring-cloud-services/linkis-service-gateway/linkis-gateway-httpclient-support/src/main/scala/org/apache/linkis/httpclient/dws/response/DWSResult.scala @@ -82,7 +82,9 @@ trait DWSResult extends Logging with HttpResult { this.url = url this.contentType = contentType } { case e: Exception => - logger.error(e.getMessage()) + throw new HttpClientResultException( + s"URL $url request failed! ResponseBody is $responseBody. ${e.getMessage}" + ) } } From d9b17724b91ab94cf2cf81e255d96bb4583c76b4 Mon Sep 17 00:00:00 2001 From: peacewong Date: Wed, 26 Apr 2023 15:36:48 +0800 Subject: [PATCH 094/261] fix Fixed session issue --- .../common/utils/ByteTimeUtilsTest.java | 47 +++++++++++++++++++ .../AccessibleExecutorConfiguration.scala | 12 +++++ .../service/DefaultAccessibleService.scala | 41 +++++++++++++++- .../heartbeat/AMHeartbeatService.scala | 1 + 4 files changed, 100 insertions(+), 1 deletion(-) create mode 100644 linkis-commons/linkis-common/src/test/java/org/apache/linkis/common/utils/ByteTimeUtilsTest.java diff --git a/linkis-commons/linkis-common/src/test/java/org/apache/linkis/common/utils/ByteTimeUtilsTest.java b/linkis-commons/linkis-common/src/test/java/org/apache/linkis/common/utils/ByteTimeUtilsTest.java new file mode 100644 index 0000000000..7b90f053ca --- /dev/null +++ b/linkis-commons/linkis-common/src/test/java/org/apache/linkis/common/utils/ByteTimeUtilsTest.java @@ -0,0 +1,47 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.common.utils; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.*; + +class ByteTimeUtilsTest { + + @Test + void byteStringAsBytes() {} + + @Test + void byteStringAsKb() {} + + @Test + void byteStringAsMb() {} + + @Test + void byteStringAsGb() { + Long res = ByteTimeUtils.byteStringAsGb("1G"); + Assertions.assertEquals(res, 1); + + Assertions.assertThrows( + NullPointerException.class, + () -> { + ByteTimeUtils.byteStringAsGb("512"); + }); + } +} diff --git a/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/org/apache/linkis/engineconn/acessible/executor/conf/AccessibleExecutorConfiguration.scala b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/org/apache/linkis/engineconn/acessible/executor/conf/AccessibleExecutorConfiguration.scala index 0eb211f731..26a25a1539 100644 --- a/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/org/apache/linkis/engineconn/acessible/executor/conf/AccessibleExecutorConfiguration.scala +++ b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/org/apache/linkis/engineconn/acessible/executor/conf/AccessibleExecutorConfiguration.scala @@ -55,4 +55,16 @@ object AccessibleExecutorConfiguration { val ENABLE_MAINTAIN_CREATORS = CommonVars("wds.linkis.engineconn.maintain.cretors", "IDE") + val REPORTING_DELAY_MS = CommonVars( + "linkis.engineconn.heartbeat.report.delay", + 20, + "Heartbeat status reporting delay, default 20ms, Negative numbers do not take effect" + ).getValue + + val REPORTING_IGNORE_MS = CommonVars( + "linkis.engineconn.heartbeat.report.ignore", + 3, + "Heartbeat status report repeated ignore, default 3ms,Negative numbers do not take effect" + ).getValue + } diff --git a/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/org/apache/linkis/engineconn/acessible/executor/service/DefaultAccessibleService.scala b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/org/apache/linkis/engineconn/acessible/executor/service/DefaultAccessibleService.scala index d07d16ce27..157c508e17 100644 --- a/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/org/apache/linkis/engineconn/acessible/executor/service/DefaultAccessibleService.scala +++ b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/org/apache/linkis/engineconn/acessible/executor/service/DefaultAccessibleService.scala @@ -19,6 +19,7 @@ package org.apache.linkis.engineconn.acessible.executor.service import org.apache.linkis.DataWorkCloudApplication import org.apache.linkis.common.utils.{Logging, Utils} +import org.apache.linkis.engineconn.acessible.executor.conf.AccessibleExecutorConfiguration import org.apache.linkis.engineconn.acessible.executor.entity.AccessibleExecutor import org.apache.linkis.engineconn.acessible.executor.listener.event.{ ExecutorCompletedEvent, @@ -57,6 +58,12 @@ class DefaultAccessibleService extends AccessibleService with Logging { private var shutDownHooked: Boolean = false + private var lastStatusChanged: Long = System.currentTimeMillis() + + private var lastStatus: NodeStatus = null + + private var lastThreadName: String = null + @Receiver override def dealEngineStopRequest( engineSuicideRequest: EngineSuicideRequest, @@ -167,7 +174,39 @@ class DefaultAccessibleService extends AccessibleService with Logging { override def onExecutorStatusChanged( executorStatusChangedEvent: ExecutorStatusChangedEvent ): Unit = { - reportHeartBeatMsg(executorStatusChangedEvent.executor) + val sinceLastTime = System.currentTimeMillis() - lastStatusChanged + val reportDelay = AccessibleExecutorConfiguration.REPORTING_DELAY_MS + if ( + reportDelay > 0 && executorStatusChangedEvent.toStatus != lastStatus && reportDelay > sinceLastTime + ) { + logger.info( + "In order to ensure that the previous state is consumed first, sleep here {} s", + reportDelay * 2 + ) + + Thread.sleep(reportDelay * 2) + } + val ignoreTime = AccessibleExecutorConfiguration.REPORTING_IGNORE_MS + val currentThreadName = Thread.currentThread().getName + if ( + ignoreTime > 0 && executorStatusChangedEvent.toStatus == lastStatus && ignoreTime > sinceLastTime && currentThreadName + .equals(lastThreadName) + ) { + logger.info( + "If the status is the same and the time is short and the thread is the same, no status report is performed {}", + executorStatusChangedEvent + ) + } else if ( + NodeStatus.Busy == lastStatus && executorStatusChangedEvent.toStatus == NodeStatus.Idle + ) { + logger.info("The state transition from Busy to Idle is not reported") + } else { + reportHeartBeatMsg(executorStatusChangedEvent.executor) + } + logger.info("Finished to report status {}", executorStatusChangedEvent) + lastStatusChanged = System.currentTimeMillis() + lastStatus = executorStatusChangedEvent.toStatus + lastThreadName = currentThreadName } private def reportHeartBeatMsg(executor: Executor): Unit = { diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/service/heartbeat/AMHeartbeatService.scala b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/service/heartbeat/AMHeartbeatService.scala index 25db0b8303..e3381985d0 100644 --- a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/service/heartbeat/AMHeartbeatService.scala +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/service/heartbeat/AMHeartbeatService.scala @@ -78,6 +78,7 @@ class AMHeartbeatService extends HeartbeatService with Logging { nodeMetrics.setStatus(0) } nodeMetricManagerPersistence.addOrupdateNodeMetrics(nodeMetrics) + logger.info(s"Finished to deal nodeHeartbeatMsg $nodeHeartbeatMsg") } } From 90a3556bc9cccb2d5243da2f62d5fd7e8bd769f9 Mon Sep 17 00:00:00 2001 From: peacewong Date: Wed, 26 Apr 2023 15:52:07 +0800 Subject: [PATCH 095/261] update to ms --- .../acessible/executor/service/DefaultAccessibleService.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/org/apache/linkis/engineconn/acessible/executor/service/DefaultAccessibleService.scala b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/org/apache/linkis/engineconn/acessible/executor/service/DefaultAccessibleService.scala index 157c508e17..06fd13b0e9 100644 --- a/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/org/apache/linkis/engineconn/acessible/executor/service/DefaultAccessibleService.scala +++ b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/org/apache/linkis/engineconn/acessible/executor/service/DefaultAccessibleService.scala @@ -180,7 +180,7 @@ class DefaultAccessibleService extends AccessibleService with Logging { reportDelay > 0 && executorStatusChangedEvent.toStatus != lastStatus && reportDelay > sinceLastTime ) { logger.info( - "In order to ensure that the previous state is consumed first, sleep here {} s", + "In order to ensure that the previous state is consumed first, sleep here {} ms", reportDelay * 2 ) From f80d82ac427da620e3fffee07eff174233cbd5b0 Mon Sep 17 00:00:00 2001 From: peacewong Date: Fri, 28 Apr 2023 13:00:33 +0800 Subject: [PATCH 096/261] add log --- .../jobhistory/service/impl/JobHistoryQueryServiceImpl.scala | 2 ++ 1 file changed, 2 insertions(+) diff --git a/linkis-public-enhancements/linkis-jobhistory/src/main/scala/org/apache/linkis/jobhistory/service/impl/JobHistoryQueryServiceImpl.scala b/linkis-public-enhancements/linkis-jobhistory/src/main/scala/org/apache/linkis/jobhistory/service/impl/JobHistoryQueryServiceImpl.scala index 6778276d3a..f00abc5568 100644 --- a/linkis-public-enhancements/linkis-jobhistory/src/main/scala/org/apache/linkis/jobhistory/service/impl/JobHistoryQueryServiceImpl.scala +++ b/linkis-public-enhancements/linkis-jobhistory/src/main/scala/org/apache/linkis/jobhistory/service/impl/JobHistoryQueryServiceImpl.scala @@ -360,6 +360,7 @@ class JobHistoryQueryServiceImpl extends JobHistoryQueryService with Logging { if (StringUtils.isBlank(cacheKey)) { getCountUndoneTasks(username, creator, sDate, eDate, engineType, startJobId) } else { + logger.info("From cache to get un Done task {}", cacheKey) unDoneTaskCache.get( cacheKey, new Callable[Integer] { @@ -379,6 +380,7 @@ class JobHistoryQueryServiceImpl extends JobHistoryQueryService with Logging { engineType: String, startJobId: lang.Long ): Integer = { + logger.info("Get count undone Tasks {}, {}, {}", username, creator, engineType) val statusList: util.List[String] = new util.ArrayList[String]() statusList.add(TaskStatus.Running.toString) statusList.add(TaskStatus.Inited.toString) From f31de8e67a940a626b50e47899b27cd6e0ca0bba Mon Sep 17 00:00:00 2001 From: peacewong Date: Mon, 8 May 2023 11:47:02 +0800 Subject: [PATCH 097/261] add license --- .../src/main/resources/version.properties | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/linkis-computation-governance/linkis-client/linkis-cli/src/main/resources/version.properties b/linkis-computation-governance/linkis-client/linkis-cli/src/main/resources/version.properties index 0888f0b359..0da37e5c03 100644 --- a/linkis-computation-governance/linkis-client/linkis-cli/src/main/resources/version.properties +++ b/linkis-computation-governance/linkis-client/linkis-cli/src/main/resources/version.properties @@ -1 +1,16 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + cli.version=${project.version} \ No newline at end of file From e5221dcef6ad774de5bb4f7c44e31e38ce74cb66 Mon Sep 17 00:00:00 2001 From: huangKai-2323 <62878639+huangKai-2323@users.noreply.github.com> Date: Mon, 8 May 2023 23:35:39 +0800 Subject: [PATCH 098/261] Dev 1.1.11 ec list add unlock time (#148) * fix class no found * fix NEP * fix NEP * code format --------- Co-authored-by: casionone --- .../manager/am/restful/ECResourceInfoRestfulApi.java | 2 +- .../am/service/impl/ECResourceInfoServiceImpl.java | 12 ++++++++---- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/restful/ECResourceInfoRestfulApi.java b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/restful/ECResourceInfoRestfulApi.java index e00a3c7bd8..65e855061d 100644 --- a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/restful/ECResourceInfoRestfulApi.java +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/restful/ECResourceInfoRestfulApi.java @@ -271,7 +271,7 @@ public Message queryEcList(HttpServletRequest req, @RequestBody JsonNode jsonNod JsonUtils.jackson() .readValue(ecInstancesParam.toString(), new TypeReference>() {}); } catch (JsonProcessingException e) { - return Message.error("parameters:instanceName parsing failed(请求参数【instanceName】解析失败)"); + return Message.error("parameters:instanceName parsing failed(请求参数【ecInstances】解析失败)"); } } logger.info( diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/service/impl/ECResourceInfoServiceImpl.java b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/service/impl/ECResourceInfoServiceImpl.java index b55d7c2630..2c00a3f84f 100644 --- a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/service/impl/ECResourceInfoServiceImpl.java +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/service/impl/ECResourceInfoServiceImpl.java @@ -161,11 +161,15 @@ public List> getECResourceInfoList( */ long lastUnlockTimestamp = 0L; if (NodeStatus.values()[intStatus].name().equals(NodeStatus.Unlock.name())) { - HashMap heartbeatMap = - BDPJettyServerHelper.gson() - .fromJson(ecNodeinfo.getHeartbeatMsg(), new HashMap<>().getClass()); + String heartbeatMsg = ecNodeinfo.getHeartbeatMsg(); + Map heartbeatMap = new HashMap<>(); + if (StringUtils.isNotBlank(heartbeatMsg)) { + heartbeatMap = + BDPJettyServerHelper.gson() + .fromJson(heartbeatMsg, new HashMap<>().getClass()); + } Object lastUnlockTimestampObject = - Optional.ofNullable(heartbeatMap.get("lastUnlockTimestamp")).orElse(0); + heartbeatMap.getOrDefault("lastUnlockTimestamp", 0); BigDecimal lastUnlockTimestampBigDecimal = new BigDecimal(String.valueOf(lastUnlockTimestampObject)); lastUnlockTimestamp = lastUnlockTimestampBigDecimal.longValue(); From 642f341190820865055a5582ebeaa3e4ee05f3f0 Mon Sep 17 00:00:00 2001 From: peacewong Date: Tue, 9 May 2023 12:04:43 +0800 Subject: [PATCH 099/261] update var name --- .../apache/linkis/ujes/jdbc/LinkisSQLConnection.scala | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/linkis-computation-governance/linkis-jdbc-driver/src/main/scala/org/apache/linkis/ujes/jdbc/LinkisSQLConnection.scala b/linkis-computation-governance/linkis-jdbc-driver/src/main/scala/org/apache/linkis/ujes/jdbc/LinkisSQLConnection.scala index 9cd64f5281..40b2704270 100644 --- a/linkis-computation-governance/linkis-jdbc-driver/src/main/scala/org/apache/linkis/ujes/jdbc/LinkisSQLConnection.scala +++ b/linkis-computation-governance/linkis-jdbc-driver/src/main/scala/org/apache/linkis/ujes/jdbc/LinkisSQLConnection.scala @@ -56,7 +56,7 @@ class LinkisSQLConnection(private[jdbc] val ujesClient: UJESClient, props: Prope private[jdbc] var creator = "JDBCDriver" - private[jdbc] var tableau = false + private[jdbc] var tableauFlag = false private[jdbc] val variableMap = { val params = props.getProperty(PARAMS) @@ -78,11 +78,11 @@ class LinkisSQLConnection(private[jdbc] val ujesClient: UJESClient, props: Prope if (params != null) { params.split(PARAM_SPLIT).map(_.split(KV_SPLIT)).foreach { case Array(TABLEAU, v) => - tableau = true + tableauFlag = true case _ => } } - tableau + tableauFlag } private[jdbc] val dbName = @@ -176,7 +176,8 @@ class LinkisSQLConnection(private[jdbc] val ujesClient: UJESClient, props: Prope override def getMetaData: DatabaseMetaData = throwWhenClosed(new UJESSQLDatabaseMetaData(this)) override def close(): Unit = { - runningSQLStatements.asScala.foreach(statement => Utils.tryQuietly(statement.close())) + runningSQLStatements.asScala.foreach{ statement => Utils.tryQuietly(statement.close()) + } closed = true } From 2142e8a4fa551d22229e9a359ca2276e3cf70ca1 Mon Sep 17 00:00:00 2001 From: peacewong Date: Tue, 9 May 2023 12:18:19 +0800 Subject: [PATCH 100/261] Fix error Status call back --- .../executor/hook/OnceEngineConnHook.scala | 5 ----- .../hook/ComputationEngineConnHook.scala | 5 ----- .../hook/CallbackEngineConnHook.scala | 22 +++++++++++++++++-- .../ujes/jdbc/LinkisSQLConnection.scala | 3 +-- 4 files changed, 21 insertions(+), 14 deletions(-) diff --git a/linkis-computation-governance/linkis-engineconn/linkis-clustered-engineconn/linkis-once-engineconn/src/main/scala/org/apache/linkis/engineconn/once/executor/hook/OnceEngineConnHook.scala b/linkis-computation-governance/linkis-engineconn/linkis-clustered-engineconn/linkis-once-engineconn/src/main/scala/org/apache/linkis/engineconn/once/executor/hook/OnceEngineConnHook.scala index cd4029bae6..2586576bb7 100644 --- a/linkis-computation-governance/linkis-engineconn/linkis-clustered-engineconn/linkis-once-engineconn/src/main/scala/org/apache/linkis/engineconn/once/executor/hook/OnceEngineConnHook.scala +++ b/linkis-computation-governance/linkis-engineconn/linkis-clustered-engineconn/linkis-once-engineconn/src/main/scala/org/apache/linkis/engineconn/once/executor/hook/OnceEngineConnHook.scala @@ -28,11 +28,6 @@ import org.apache.linkis.manager.common.entity.enumeration.NodeStatus */ class OnceEngineConnHook extends CallbackEngineConnHook { - override protected def getNodeStatusOfStartSuccess( - engineCreationContext: EngineCreationContext, - engineConn: EngineConn - ): NodeStatus = NodeStatus.Unlock - override def afterEngineServerStartSuccess( engineCreationContext: EngineCreationContext, engineConn: EngineConn diff --git a/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/hook/ComputationEngineConnHook.scala b/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/hook/ComputationEngineConnHook.scala index d22bf3f800..e5ccd2bfbe 100644 --- a/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/hook/ComputationEngineConnHook.scala +++ b/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/hook/ComputationEngineConnHook.scala @@ -25,11 +25,6 @@ import org.apache.linkis.manager.common.entity.enumeration.NodeStatus class ComputationEngineConnHook extends CallbackEngineConnHook { - override protected def getNodeStatusOfStartSuccess( - engineCreationContext: EngineCreationContext, - engineConn: EngineConn - ): NodeStatus = NodeStatus.Unlock - override def afterEngineServerStartSuccess( engineCreationContext: EngineCreationContext, engineConn: EngineConn diff --git a/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/org/apache/linkis/engineconn/callback/hook/CallbackEngineConnHook.scala b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/org/apache/linkis/engineconn/callback/hook/CallbackEngineConnHook.scala index d7ad2c7979..6c9e1e68f4 100644 --- a/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/org/apache/linkis/engineconn/callback/hook/CallbackEngineConnHook.scala +++ b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/org/apache/linkis/engineconn/callback/hook/CallbackEngineConnHook.scala @@ -20,6 +20,7 @@ package org.apache.linkis.engineconn.callback.hook import org.apache.linkis.DataWorkCloudApplication import org.apache.linkis.common.conf.DWCArgumentsParser import org.apache.linkis.common.utils.{Logging, Utils} +import org.apache.linkis.engineconn.acessible.executor.entity.AccessibleExecutor import org.apache.linkis.engineconn.callback.service.{ EngineConnAfterStartCallback, EngineConnPidCallback @@ -28,6 +29,7 @@ import org.apache.linkis.engineconn.common.conf.EngineConnConf import org.apache.linkis.engineconn.common.creation.EngineCreationContext import org.apache.linkis.engineconn.common.engineconn.EngineConn import org.apache.linkis.engineconn.common.hook.EngineConnHook +import org.apache.linkis.engineconn.core.executor.ExecutorManager import org.apache.linkis.engineconn.core.hook.ShutdownHook import org.apache.linkis.manager.common.entity.enumeration.NodeStatus import org.apache.linkis.manager.common.protocol.engine.EngineConnStatusCallback @@ -88,7 +90,7 @@ class CallbackEngineConnHook extends EngineConnHook with Logging { Sender.getThisServiceInstance, engineCreationContext.getTicketId, NodeStatus.Failed, - prefixMsg + ExceptionUtils.getRootCauseMessage(throwable) + prefixMsg + ExceptionUtils.getStackTrace(throwable) ) ) ) @@ -99,7 +101,23 @@ class CallbackEngineConnHook extends EngineConnHook with Logging { protected def getNodeStatusOfStartSuccess( engineCreationContext: EngineCreationContext, engineConn: EngineConn - ): NodeStatus = NodeStatus.Success + ): NodeStatus = { + ExecutorManager.getInstance.getReportExecutor match { + case executor: AccessibleExecutor => + if ( + executor.getStatus == NodeStatus.ShuttingDown || executor.getStatus == NodeStatus.Failed + ) { + logger.info( + "The status of EngineConn is {}, and the actual status will be reported", + executor.getStatus + ) + executor.getStatus + } else { + NodeStatus.Unlock + } + case _ => NodeStatus.Unlock + } + } override def afterEngineServerStartSuccess( engineCreationContext: EngineCreationContext, diff --git a/linkis-computation-governance/linkis-jdbc-driver/src/main/scala/org/apache/linkis/ujes/jdbc/LinkisSQLConnection.scala b/linkis-computation-governance/linkis-jdbc-driver/src/main/scala/org/apache/linkis/ujes/jdbc/LinkisSQLConnection.scala index 40b2704270..6683161b8e 100644 --- a/linkis-computation-governance/linkis-jdbc-driver/src/main/scala/org/apache/linkis/ujes/jdbc/LinkisSQLConnection.scala +++ b/linkis-computation-governance/linkis-jdbc-driver/src/main/scala/org/apache/linkis/ujes/jdbc/LinkisSQLConnection.scala @@ -176,8 +176,7 @@ class LinkisSQLConnection(private[jdbc] val ujesClient: UJESClient, props: Prope override def getMetaData: DatabaseMetaData = throwWhenClosed(new UJESSQLDatabaseMetaData(this)) override def close(): Unit = { - runningSQLStatements.asScala.foreach{ statement => Utils.tryQuietly(statement.close()) - } + runningSQLStatements.asScala.foreach { statement => Utils.tryQuietly(statement.close()) } closed = true } From d7566aaf9f5d9bec5d8856eb2d32692a9339e38b Mon Sep 17 00:00:00 2001 From: peacewong Date: Tue, 9 May 2023 14:19:09 +0800 Subject: [PATCH 101/261] Dev 1.4.0 jdbc optimize (#149) * Optimize jdbc driver * fix Fixed session issue * update to ms * add log * add license * update var name * Fix error Status call back --- .../config/ClientConfigBuilder.scala | 8 +- .../src/main/resources/version.properties | 15 ++ .../ujes/client/response/OpenLogResult.scala | 2 +- .../executor/hook/OnceEngineConnHook.scala | 5 - .../hook/ComputationEngineConnHook.scala | 5 - .../hook/CallbackEngineConnHook.scala | 22 ++- .../linkis/ujes/jdbc/UJESSQLDriver.java | 2 +- .../linkis/ujes/jdbc/utils/JDBCUtils.java | 2 - ...ection.scala => LinkisSQLConnection.scala} | 130 ++++++++++---- ...scala => LinkisSQLPreparedStatement.scala} | 4 +- ...atement.scala => LinkisSQLStatement.scala} | 166 ++++++++++-------- .../linkis/ujes/jdbc/UJESClientFactory.scala | 22 ++- .../ujes/jdbc/UJESSQLDatabaseMetaData.scala | 19 +- .../linkis/ujes/jdbc/UJESSQLDriverMain.scala | 8 +- .../linkis/ujes/jdbc/UJESSQLResultSet.scala | 28 +-- .../linkis/ujes/jdbc/UJESSQLTypeParser.scala | 11 +- .../hook/JDBCDriverPreExecutionHook.scala | 4 +- .../jdbc/hook/impl/NoLimitExecutionHook.scala | 6 +- .../hook/impl/TableauPreExecutionHook.scala | 5 +- .../linkis/ujes/jdbc/CreateConnection.java | 6 +- .../apache/linkis/ujes/jdbc/JDBCSpiTest.java | 12 +- ...tTest.java => LinkisSQLStatementTest.java} | 8 +- .../jdbc/UJESSQLDatabaseMetaDataTest.java | 2 +- .../jdbc/UJESSQLPreparedStatementTest.java | 4 +- .../ujes/jdbc/UJESSQLResultSetTest.java | 4 +- .../httpclient/dws/response/DWSResult.scala | 4 +- 26 files changed, 326 insertions(+), 178 deletions(-) rename linkis-computation-governance/linkis-jdbc-driver/src/main/scala/org/apache/linkis/ujes/jdbc/{UJESSQLConnection.scala => LinkisSQLConnection.scala} (73%) rename linkis-computation-governance/linkis-jdbc-driver/src/main/scala/org/apache/linkis/ujes/jdbc/{UJESSQLPreparedStatement.scala => LinkisSQLPreparedStatement.scala} (98%) rename linkis-computation-governance/linkis-jdbc-driver/src/main/scala/org/apache/linkis/ujes/jdbc/{UJESSQLStatement.scala => LinkisSQLStatement.scala} (72%) rename linkis-computation-governance/linkis-jdbc-driver/src/test/java/org/apache/linkis/ujes/jdbc/{UJESSQLStatementTest.java => LinkisSQLStatementTest.java} (95%) diff --git a/linkis-commons/linkis-httpclient/src/main/scala/org/apache/linkis/httpclient/config/ClientConfigBuilder.scala b/linkis-commons/linkis-httpclient/src/main/scala/org/apache/linkis/httpclient/config/ClientConfigBuilder.scala index c3e5afba30..b1fc579f3c 100644 --- a/linkis-commons/linkis-httpclient/src/main/scala/org/apache/linkis/httpclient/config/ClientConfigBuilder.scala +++ b/linkis-commons/linkis-httpclient/src/main/scala/org/apache/linkis/httpclient/config/ClientConfigBuilder.scala @@ -17,6 +17,7 @@ package org.apache.linkis.httpclient.config +import org.apache.linkis.common.exception.LinkisRetryException import org.apache.linkis.common.utils.{DefaultRetryHandler, RetryHandler} import org.apache.linkis.httpclient.authentication.AuthenticationStrategy import org.apache.linkis.httpclient.loadbalancer.LoadBalancerStrategy @@ -38,7 +39,12 @@ class ClientConfigBuilder protected () { protected var readTimeout: Long = _ protected var maxConnection: Int = _ protected var retryEnabled: Boolean = true - protected var retryHandler: RetryHandler = new DefaultRetryHandler + + protected var retryHandler: RetryHandler = { + val retryHandler = new DefaultRetryHandler + retryHandler.addRetryException(classOf[LinkisRetryException]) + retryHandler + } def addServerUrl(serverUrl: String): this.type = { this.serverUrl = serverUrl diff --git a/linkis-computation-governance/linkis-client/linkis-cli/src/main/resources/version.properties b/linkis-computation-governance/linkis-client/linkis-cli/src/main/resources/version.properties index 0888f0b359..0da37e5c03 100644 --- a/linkis-computation-governance/linkis-client/linkis-cli/src/main/resources/version.properties +++ b/linkis-computation-governance/linkis-client/linkis-cli/src/main/resources/version.properties @@ -1 +1,16 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + cli.version=${project.version} \ No newline at end of file diff --git a/linkis-computation-governance/linkis-client/linkis-computation-client/src/main/scala/org/apache/linkis/ujes/client/response/OpenLogResult.scala b/linkis-computation-governance/linkis-client/linkis-computation-client/src/main/scala/org/apache/linkis/ujes/client/response/OpenLogResult.scala index fe107a32ca..2de5758aea 100644 --- a/linkis-computation-governance/linkis-client/linkis-computation-client/src/main/scala/org/apache/linkis/ujes/client/response/OpenLogResult.scala +++ b/linkis-computation-governance/linkis-client/linkis-computation-client/src/main/scala/org/apache/linkis/ujes/client/response/OpenLogResult.scala @@ -26,7 +26,7 @@ import scala.beans.BeanProperty class OpenLogResult extends DWSResult { /** - * log[0] - info log[1] - warn log[2] - error log[3] - all (info + warn + error) + * log[0] - error log[1] - warn log[2] - info log[3] - all (info + warn + error) */ @BeanProperty var log: Array[String] = _ diff --git a/linkis-computation-governance/linkis-engineconn/linkis-clustered-engineconn/linkis-once-engineconn/src/main/scala/org/apache/linkis/engineconn/once/executor/hook/OnceEngineConnHook.scala b/linkis-computation-governance/linkis-engineconn/linkis-clustered-engineconn/linkis-once-engineconn/src/main/scala/org/apache/linkis/engineconn/once/executor/hook/OnceEngineConnHook.scala index cd4029bae6..2586576bb7 100644 --- a/linkis-computation-governance/linkis-engineconn/linkis-clustered-engineconn/linkis-once-engineconn/src/main/scala/org/apache/linkis/engineconn/once/executor/hook/OnceEngineConnHook.scala +++ b/linkis-computation-governance/linkis-engineconn/linkis-clustered-engineconn/linkis-once-engineconn/src/main/scala/org/apache/linkis/engineconn/once/executor/hook/OnceEngineConnHook.scala @@ -28,11 +28,6 @@ import org.apache.linkis.manager.common.entity.enumeration.NodeStatus */ class OnceEngineConnHook extends CallbackEngineConnHook { - override protected def getNodeStatusOfStartSuccess( - engineCreationContext: EngineCreationContext, - engineConn: EngineConn - ): NodeStatus = NodeStatus.Unlock - override def afterEngineServerStartSuccess( engineCreationContext: EngineCreationContext, engineConn: EngineConn diff --git a/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/hook/ComputationEngineConnHook.scala b/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/hook/ComputationEngineConnHook.scala index d22bf3f800..e5ccd2bfbe 100644 --- a/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/hook/ComputationEngineConnHook.scala +++ b/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/hook/ComputationEngineConnHook.scala @@ -25,11 +25,6 @@ import org.apache.linkis.manager.common.entity.enumeration.NodeStatus class ComputationEngineConnHook extends CallbackEngineConnHook { - override protected def getNodeStatusOfStartSuccess( - engineCreationContext: EngineCreationContext, - engineConn: EngineConn - ): NodeStatus = NodeStatus.Unlock - override def afterEngineServerStartSuccess( engineCreationContext: EngineCreationContext, engineConn: EngineConn diff --git a/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/org/apache/linkis/engineconn/callback/hook/CallbackEngineConnHook.scala b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/org/apache/linkis/engineconn/callback/hook/CallbackEngineConnHook.scala index d7ad2c7979..6c9e1e68f4 100644 --- a/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/org/apache/linkis/engineconn/callback/hook/CallbackEngineConnHook.scala +++ b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/org/apache/linkis/engineconn/callback/hook/CallbackEngineConnHook.scala @@ -20,6 +20,7 @@ package org.apache.linkis.engineconn.callback.hook import org.apache.linkis.DataWorkCloudApplication import org.apache.linkis.common.conf.DWCArgumentsParser import org.apache.linkis.common.utils.{Logging, Utils} +import org.apache.linkis.engineconn.acessible.executor.entity.AccessibleExecutor import org.apache.linkis.engineconn.callback.service.{ EngineConnAfterStartCallback, EngineConnPidCallback @@ -28,6 +29,7 @@ import org.apache.linkis.engineconn.common.conf.EngineConnConf import org.apache.linkis.engineconn.common.creation.EngineCreationContext import org.apache.linkis.engineconn.common.engineconn.EngineConn import org.apache.linkis.engineconn.common.hook.EngineConnHook +import org.apache.linkis.engineconn.core.executor.ExecutorManager import org.apache.linkis.engineconn.core.hook.ShutdownHook import org.apache.linkis.manager.common.entity.enumeration.NodeStatus import org.apache.linkis.manager.common.protocol.engine.EngineConnStatusCallback @@ -88,7 +90,7 @@ class CallbackEngineConnHook extends EngineConnHook with Logging { Sender.getThisServiceInstance, engineCreationContext.getTicketId, NodeStatus.Failed, - prefixMsg + ExceptionUtils.getRootCauseMessage(throwable) + prefixMsg + ExceptionUtils.getStackTrace(throwable) ) ) ) @@ -99,7 +101,23 @@ class CallbackEngineConnHook extends EngineConnHook with Logging { protected def getNodeStatusOfStartSuccess( engineCreationContext: EngineCreationContext, engineConn: EngineConn - ): NodeStatus = NodeStatus.Success + ): NodeStatus = { + ExecutorManager.getInstance.getReportExecutor match { + case executor: AccessibleExecutor => + if ( + executor.getStatus == NodeStatus.ShuttingDown || executor.getStatus == NodeStatus.Failed + ) { + logger.info( + "The status of EngineConn is {}, and the actual status will be reported", + executor.getStatus + ) + executor.getStatus + } else { + NodeStatus.Unlock + } + case _ => NodeStatus.Unlock + } + } override def afterEngineServerStartSuccess( engineCreationContext: EngineCreationContext, diff --git a/linkis-computation-governance/linkis-jdbc-driver/src/main/java/org/apache/linkis/ujes/jdbc/UJESSQLDriver.java b/linkis-computation-governance/linkis-jdbc-driver/src/main/java/org/apache/linkis/ujes/jdbc/UJESSQLDriver.java index 90e1f73563..392f4dd097 100644 --- a/linkis-computation-governance/linkis-jdbc-driver/src/main/java/org/apache/linkis/ujes/jdbc/UJESSQLDriver.java +++ b/linkis-computation-governance/linkis-jdbc-driver/src/main/java/org/apache/linkis/ujes/jdbc/UJESSQLDriver.java @@ -58,7 +58,7 @@ public class UJESSQLDriver extends UJESSQLDriverMain implements Driver { static String ENABLE_DISCOVERY = "enableDiscovery"; static String ENABLE_LOADBALANCER = "enableLoadBalancer"; static String CREATOR = "creator"; - + static String TABLEAU = "tableau"; static String VARIABLE_HEADER = "var:"; static String PARAM_SPLIT = "&"; static String KV_SPLIT = "="; diff --git a/linkis-computation-governance/linkis-jdbc-driver/src/main/java/org/apache/linkis/ujes/jdbc/utils/JDBCUtils.java b/linkis-computation-governance/linkis-jdbc-driver/src/main/java/org/apache/linkis/ujes/jdbc/utils/JDBCUtils.java index 061acd42f9..8f7dcfed12 100644 --- a/linkis-computation-governance/linkis-jdbc-driver/src/main/java/org/apache/linkis/ujes/jdbc/utils/JDBCUtils.java +++ b/linkis-computation-governance/linkis-jdbc-driver/src/main/java/org/apache/linkis/ujes/jdbc/utils/JDBCUtils.java @@ -41,8 +41,6 @@ public static String convertPattern(final String pattern) { continue; } else if (c == '%') { result.append(".*"); - } else if (c == '_') { - result.append('.'); } else { result.append(Character.toLowerCase(c)); } diff --git a/linkis-computation-governance/linkis-jdbc-driver/src/main/scala/org/apache/linkis/ujes/jdbc/UJESSQLConnection.scala b/linkis-computation-governance/linkis-jdbc-driver/src/main/scala/org/apache/linkis/ujes/jdbc/LinkisSQLConnection.scala similarity index 73% rename from linkis-computation-governance/linkis-jdbc-driver/src/main/scala/org/apache/linkis/ujes/jdbc/UJESSQLConnection.scala rename to linkis-computation-governance/linkis-jdbc-driver/src/main/scala/org/apache/linkis/ujes/jdbc/LinkisSQLConnection.scala index 0d8403c274..6683161b8e 100644 --- a/linkis-computation-governance/linkis-jdbc-driver/src/main/scala/org/apache/linkis/ujes/jdbc/UJESSQLConnection.scala +++ b/linkis-computation-governance/linkis-jdbc-driver/src/main/scala/org/apache/linkis/ujes/jdbc/LinkisSQLConnection.scala @@ -18,8 +18,12 @@ package org.apache.linkis.ujes.jdbc import org.apache.linkis.common.utils.{Logging, Utils} +import org.apache.linkis.manager.label.constant.LabelKeyConstant +import org.apache.linkis.manager.label.entity.engine.{EngineType, EngineTypeLabel, RunType} +import org.apache.linkis.manager.label.utils.EngineTypeLabelCreator import org.apache.linkis.ujes.client.UJESClient -import org.apache.linkis.ujes.client.request.JobExecuteAction.EngineType +import org.apache.linkis.ujes.client.request.JobSubmitAction +import org.apache.linkis.ujes.client.response.JobExecuteResult import org.apache.linkis.ujes.jdbc.UJESSQLDriverMain._ import org.apache.commons.lang3.StringUtils @@ -44,26 +48,41 @@ import java.sql.{ import java.util.Properties import java.util.concurrent.Executor -import scala.collection.{mutable, JavaConversions} +import scala.collection.JavaConverters._ -class UJESSQLConnection(private[jdbc] val ujesClient: UJESClient, props: Properties) +class LinkisSQLConnection(private[jdbc] val ujesClient: UJESClient, props: Properties) extends Connection with Logging { - private[jdbc] var creator = "IDE" + + private[jdbc] var creator = "JDBCDriver" + + private[jdbc] var tableauFlag = false private[jdbc] val variableMap = { val params = props.getProperty(PARAMS) - val map = new mutable.HashMap[String, AnyRef] + val map = new util.HashMap[String, AnyRef] if (params != null) { params.split(PARAM_SPLIT).map(_.split(KV_SPLIT)).foreach { case Array(k, v) if k.startsWith(VARIABLE_HEADER) => - map += k.substring(VARIABLE_HEADER.length) -> v + map.put(k.substring(VARIABLE_HEADER.length), v) case Array(CREATOR, v) => creator = v case _ => } } - map.toMap + map + } + + def isTableau(): Boolean = { + val params = props.getProperty(PARAMS) + if (params != null) { + params.split(PARAM_SPLIT).map(_.split(KV_SPLIT)).foreach { + case Array(TABLEAU, v) => + tableauFlag = true + case _ => + } + } + tableauFlag } private[jdbc] val dbName = @@ -80,23 +99,21 @@ class UJESSQLConnection(private[jdbc] val ujesClient: UJESClient, props: Propert private[jdbc] val serverURL = props.getProperty("URL") - private val engineTypeMap: mutable.HashMap[String, EngineType] = new mutable.HashMap() + private val labelMap: util.Map[String, AnyRef] = new util.HashMap[String, AnyRef] - private[jdbc] def getEngineType: EngineType = { - if (engineTypeMap.isEmpty) { - engineTypeMap.put(EngineType.SPARK.toString, EngineType.SPARK) - engineTypeMap.put(EngineType.HIVE.toString, EngineType.HIVE) - engineTypeMap.put(EngineType.JDBC.toString, EngineType.JDBC) - engineTypeMap.put(EngineType.PYTHON.toString, EngineType.PYTHON) - engineTypeMap.put(EngineType.SHELL.toString, EngineType.SHELL) - engineTypeMap.put(EngineType.PRESTO.toString, EngineType.PRESTO) - } - val engineType: EngineType = EngineType.PRESTO + private val startupParams: util.Map[String, AnyRef] = new util.HashMap[String, AnyRef] + + private val runtimeParams: util.Map[String, AnyRef] = new util.HashMap[String, AnyRef] + + private[jdbc] def getEngineType: EngineTypeLabel = { + val engineType: EngineTypeLabel = + EngineTypeLabelCreator.createEngineTypeLabel(EngineType.TRINO.toString) if (props.containsKey(PARAMS)) { val params = props.getProperty(PARAMS) if (params != null & params.length() > 0) { params.split(PARAM_SPLIT).map(_.split(KV_SPLIT)).foreach { - case Array(k, v) if k.equals(UJESSQLDriver.ENGINE_TYPE) => return engineTypeMap(v) + case Array(k, v) if k.equals(UJESSQLDriver.ENGINE_TYPE) => + return EngineTypeLabelCreator.createEngineTypeLabel(v) case _ => } } @@ -121,28 +138,30 @@ class UJESSQLConnection(private[jdbc] val ujesClient: UJESClient, props: Propert def getProps: Properties = props - def removeStatement(statement: UJESSQLStatement): Unit = runningSQLStatements.remove(statement) + def removeStatement(statement: LinkisSQLStatement): Unit = runningSQLStatements.remove(statement) - override def createStatement(): Statement = createStatementAndAdd(new UJESSQLStatement(this)) + override def createStatement(): Statement = createStatementAndAdd(new LinkisSQLStatement(this)) - override def prepareStatement(sql: String): UJESSQLPreparedStatement = { - val statement = createStatementAndAdd(new UJESSQLPreparedStatement(this, sql)) + override def prepareStatement(sql: String): LinkisSQLPreparedStatement = { + val statement = createStatementAndAdd(new LinkisSQLPreparedStatement(this, sql)) statement.clearQuery() statement } override def createStatement(resultSetType: Int, resultSetConcurrency: Int): Statement = { - if (resultSetConcurrency != ResultSet.CONCUR_READ_ONLY) + if (resultSetConcurrency != ResultSet.CONCUR_READ_ONLY) { throw new SQLException( "Statement with resultset concurrency " + resultSetConcurrency + " is not supported", "HYC00" ) - if (resultSetType == ResultSet.TYPE_SCROLL_SENSITIVE) + } + if (resultSetType == ResultSet.TYPE_SCROLL_SENSITIVE) { throw new SQLException( "Statement with resultset type " + resultSetType + " is not supported", "HYC00" ) - createStatementAndAdd(new UJESSQLStatement(this)) + } + createStatementAndAdd(new LinkisSQLStatement(this)) } override def prepareStatement(sql: String, autoGeneratedKeys: Int): PreparedStatement = @@ -157,9 +176,7 @@ class UJESSQLConnection(private[jdbc] val ujesClient: UJESClient, props: Propert override def getMetaData: DatabaseMetaData = throwWhenClosed(new UJESSQLDatabaseMetaData(this)) override def close(): Unit = { - JavaConversions - .asScalaBuffer(runningSQLStatements) - .foreach(statement => Utils.tryQuietly(statement.close())) + runningSQLStatements.asScala.foreach { statement => Utils.tryQuietly(statement.close()) } closed = true } @@ -305,15 +322,17 @@ class UJESSQLConnection(private[jdbc] val ujesClient: UJESClient, props: Propert throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_CONNECTION, "createStruct not supported") override def setSchema(schema: String): Unit = throwWhenClosed { - if (StringUtils.isBlank(schema)) + if (StringUtils.isBlank(schema)) { throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_STATEMENT, "schema is empty!") + } createStatement().execute("use " + schema) } override def getSchema: String = throwWhenClosed { val resultSet = createStatement().executeQuery("SELECT current_database()") - if (!resultSet.next()) + if (!resultSet.next()) { throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_STATEMENT, "Get schema failed!") + } resultSet.getString(1) } @@ -337,4 +356,53 @@ class UJESSQLConnection(private[jdbc] val ujesClient: UJESClient, props: Propert override def isWrapperFor(iface: Class[_]): Boolean = throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_CONNECTION, "isWrapperFor not supported") + def addLabels(labels: util.Map[String, AnyRef]): Unit = { + labelMap.putAll(labels) + } + + def addStartUpParams(params: util.Map[String, AnyRef]): Unit = { + startupParams.putAll(params) + } + + def addRuntimeParams(params: util.Map[String, AnyRef]): Unit = { + runtimeParams.putAll(params) + } + + def engineToCodeType(engine: String): String = { + val runType = EngineType.mapStringToEngineType(engine) match { + case EngineType.SPARK => RunType.SQL + case EngineType.HIVE => RunType.HIVE + case EngineType.TRINO => RunType.TRINO_SQL + case EngineType.PRESTO => RunType.PRESTO_SQL + case EngineType.ELASTICSEARCH => RunType.ES_SQL + case EngineType.JDBC => RunType.JDBC + case EngineType.PYTHON => RunType.SHELL + case _ => RunType.SQL + } + runType.toString + } + + private[jdbc] def toSubmit(code: String): JobExecuteResult = { + val engineTypeLabel = getEngineType + labelMap.put(LabelKeyConstant.ENGINE_TYPE_KEY, engineTypeLabel.getStringValue) + labelMap.put(LabelKeyConstant.USER_CREATOR_TYPE_KEY, s"$user-$creator") + labelMap.put(LabelKeyConstant.CODE_TYPE_KEY, engineToCodeType(engineTypeLabel.getEngineType)) + + val jobSubmitAction = JobSubmitAction.builder + .addExecuteCode(code) + .setStartupParams(startupParams) + .setUser(user) + .addExecuteUser(user) + .setLabels(labelMap) + .setRuntimeParams(runtimeParams) + .setVariableMap(variableMap) + .build + + val result = ujesClient.submit(jobSubmitAction) + if (result.getStatus != 0) { + throw new SQLException(result.getMessage) + } + result + } + } diff --git a/linkis-computation-governance/linkis-jdbc-driver/src/main/scala/org/apache/linkis/ujes/jdbc/UJESSQLPreparedStatement.scala b/linkis-computation-governance/linkis-jdbc-driver/src/main/scala/org/apache/linkis/ujes/jdbc/LinkisSQLPreparedStatement.scala similarity index 98% rename from linkis-computation-governance/linkis-jdbc-driver/src/main/scala/org/apache/linkis/ujes/jdbc/UJESSQLPreparedStatement.scala rename to linkis-computation-governance/linkis-jdbc-driver/src/main/scala/org/apache/linkis/ujes/jdbc/LinkisSQLPreparedStatement.scala index 6328da99e2..326a16bc7c 100644 --- a/linkis-computation-governance/linkis-jdbc-driver/src/main/scala/org/apache/linkis/ujes/jdbc/UJESSQLPreparedStatement.scala +++ b/linkis-computation-governance/linkis-jdbc-driver/src/main/scala/org/apache/linkis/ujes/jdbc/LinkisSQLPreparedStatement.scala @@ -36,8 +36,8 @@ import java.sql.{ import java.util import java.util.Calendar -class UJESSQLPreparedStatement(ujesSQLConnection: UJESSQLConnection, sql: String) - extends UJESSQLStatement(ujesSQLConnection) +class LinkisSQLPreparedStatement(ujesSQLConnection: LinkisSQLConnection, sql: String) + extends LinkisSQLStatement(ujesSQLConnection) with PreparedStatement { private val parameters = new util.HashMap[Int, Any] diff --git a/linkis-computation-governance/linkis-jdbc-driver/src/main/scala/org/apache/linkis/ujes/jdbc/UJESSQLStatement.scala b/linkis-computation-governance/linkis-jdbc-driver/src/main/scala/org/apache/linkis/ujes/jdbc/LinkisSQLStatement.scala similarity index 72% rename from linkis-computation-governance/linkis-jdbc-driver/src/main/scala/org/apache/linkis/ujes/jdbc/UJESSQLStatement.scala rename to linkis-computation-governance/linkis-jdbc-driver/src/main/scala/org/apache/linkis/ujes/jdbc/LinkisSQLStatement.scala index 916bdbaa93..1f6d5fc114 100644 --- a/linkis-computation-governance/linkis-jdbc-driver/src/main/scala/org/apache/linkis/ujes/jdbc/UJESSQLStatement.scala +++ b/linkis-computation-governance/linkis-jdbc-driver/src/main/scala/org/apache/linkis/ujes/jdbc/LinkisSQLStatement.scala @@ -19,19 +19,21 @@ package org.apache.linkis.ujes.jdbc import org.apache.linkis.common.exception.ErrorException import org.apache.linkis.common.utils.{Logging, Utils} -import org.apache.linkis.ujes.client.request.JobExecuteAction -import org.apache.linkis.ujes.client.request.JobExecuteAction.EngineType +import org.apache.linkis.governance.common.entity.ExecutionNodeStatus +import org.apache.linkis.ujes.client.request.OpenLogAction import org.apache.linkis.ujes.client.response.JobExecuteResult import org.apache.linkis.ujes.jdbc.hook.JDBCDriverPreExecutionHook +import org.apache.commons.lang3.StringUtils + import java.sql.{Connection, ResultSet, SQLWarning, Statement} +import java.util import java.util.concurrent.TimeUnit -import scala.collection.JavaConverters.mapAsJavaMapConverter import scala.concurrent.TimeoutException import scala.concurrent.duration.Duration -class UJESSQLStatement(private[jdbc] val ujesSQLConnection: UJESSQLConnection) +class LinkisSQLStatement(private[jdbc] val ujesSQLConnection: LinkisSQLConnection) extends Statement with Logging { @@ -42,8 +44,13 @@ class UJESSQLStatement(private[jdbc] val ujesSQLConnection: UJESSQLConnection) private var fetchSize = 100 private var queryTimeout = 0 + private var logPath: String = null + private var queryEnd = false + private var logFromLen = 0 + private val logSize = 100 + private[jdbc] def throwWhenClosed[T](op: => T): T = ujesSQLConnection.throwWhenClosed { if (isClosed) throw new UJESSQLException(UJESSQLErrorCode.STATEMENT_CLOSED) else op @@ -89,11 +96,12 @@ class UJESSQLStatement(private[jdbc] val ujesSQLConnection: UJESSQLConnection) override def setMaxRows(max: Int): Unit = this.maxRows = max - override def setEscapeProcessing(enable: Boolean): Unit = if (enable) + override def setEscapeProcessing(enable: Boolean): Unit = if (enable) { throw new UJESSQLException( UJESSQLErrorCode.NOSUPPORT_STATEMENT, "setEscapeProcessing not supported" ) + } override def getQueryTimeout: Int = queryTimeout @@ -108,88 +116,62 @@ class UJESSQLStatement(private[jdbc] val ujesSQLConnection: UJESSQLConnection) override def setCursorName(name: String): Unit = throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_STATEMENT, "setCursorName not supported") - override def execute(sql: String): Boolean = Utils.tryCatch(throwWhenClosed { + override def execute(sql: String): Boolean = throwWhenClosed { var parsedSQL = sql JDBCDriverPreExecutionHook.getPreExecutionHooks.foreach { preExecution => - parsedSQL = preExecution.callPreExecutionHook(parsedSQL) + parsedSQL = preExecution.callPreExecutionHook(parsedSQL, !ujesSQLConnection.isTableau()) } logger.info(s"begin to execute sql ${parsedSQL}") - val action = JobExecuteAction - .builder() - .setEngineType(ujesSQLConnection.getEngineType) - .addExecuteCode(parsedSQL) - .setCreator(ujesSQLConnection.creator) - .setUser(ujesSQLConnection.user) - if (ujesSQLConnection.variableMap.nonEmpty) { - action.setVariableMap(ujesSQLConnection.variableMap.asJava) - } - jobExecuteResult = - Utils.tryCatch(ujesSQLConnection.ujesClient.execute(action.build())) { t: Throwable => - logger.error("UJESClient failed to get result", t) - null - } - // jobExecuteResult = ujesSQLConnection.ujesClient.execute(action.build()) queryEnd = false - var status = ujesSQLConnection.ujesClient.status(jobExecuteResult) - val atMost = - if (queryTimeout > 0) Duration(queryTimeout, TimeUnit.MILLISECONDS) else Duration.Inf - if (!status.isCompleted) Utils.tryThrow { - Utils.waitUntil( - () => { - status = ujesSQLConnection.ujesClient.status(jobExecuteResult) - status.isCompleted || closed - }, - atMost, - 100, - 10000 - ) - } { - case t: TimeoutException => - if (queryTimeout > 0) clearQuery() - new UJESSQLException(UJESSQLErrorCode.QUERY_TIMEOUT, "query has been timeout!").initCause(t) - case t => t - } - if (!closed) { + logPath = null + Utils.tryFinally { + jobExecuteResult = ujesSQLConnection.toSubmit(parsedSQL) + val atMost = + if (queryTimeout > 0) Duration(queryTimeout, TimeUnit.MILLISECONDS) else Duration.Inf var jobInfo = ujesSQLConnection.ujesClient.getJobInfo(jobExecuteResult) - if (status.isFailed) + logPath = jobInfo.getRequestPersistTask.getLogPath + if (!ExecutionNodeStatus.isCompleted(ExecutionNodeStatus.valueOf(jobInfo.getJobStatus))) { + Utils.tryThrow { + Utils.waitUntil( + () => { + jobInfo = ujesSQLConnection.ujesClient.getJobInfo(jobExecuteResult) + ExecutionNodeStatus.isCompleted( + ExecutionNodeStatus.valueOf(jobInfo.getJobStatus) + ) || closed + }, + atMost, + 100, + 10000 + ) + } { + case t: TimeoutException => + if (queryTimeout > 0) clearQuery() + logPath = jobInfo.getRequestPersistTask.getLogPath + new UJESSQLException(UJESSQLErrorCode.QUERY_TIMEOUT, "query has been timeout!") + .initCause(t) + case t => t + } + } + logPath = jobInfo.getRequestPersistTask.getLogPath + if (!ExecutionNodeStatus.isSucceed(ExecutionNodeStatus.valueOf(jobInfo.getJobStatus))) { throw new ErrorException( jobInfo.getRequestPersistTask.getErrCode, jobInfo.getRequestPersistTask.getErrDesc ) - val jobInfoStatus = jobInfo.getJobStatus - if (!jobInfoStatus.equals("Succeed")) Utils.tryThrow { - Utils.waitUntil( - () => { - jobInfo = ujesSQLConnection.ujesClient.getJobInfo(jobExecuteResult) - val state = jobInfo.getJobStatus match { - case "Failed" | "Cancelled" | "Timeout" | "Succeed" => true - case _ => false - } - state || closed - }, - atMost, - 100, - 10000 - ) - } { - case t: TimeoutException => - if (queryTimeout > 0) clearQuery() - new UJESSQLException(UJESSQLErrorCode.QUERY_TIMEOUT, "query has been timeout!") - .initCause(t) - case t => t } + logger.info(s"end to execute sql ${parsedSQL}") val resultSetList = jobInfo.getResultSetList(ujesSQLConnection.ujesClient) logger.info(s"resultSetList is ${resultSetList.mkString(",")}") - queryEnd = true - if (resultSetList != null) { + if (resultSetList != null && resultSetList.nonEmpty) { resultSet = new UJESSQLResultSet(resultSetList, this, maxRows, fetchSize) true - } else false - } else throw new UJESSQLException(UJESSQLErrorCode.STATEMENT_CLOSED, "Statement is closed.") - }) { case t: Throwable => - logger.error("任务执行失败", t) - false + } else { + false + } + } { + queryEnd = true + } } def getJobExcuteResult: JobExecuteResult = jobExecuteResult @@ -201,13 +183,12 @@ class UJESSQLStatement(private[jdbc] val ujesSQLConnection: UJESSQLConnection) override def getMoreResults: Boolean = false override def setFetchDirection(direction: Int): Unit = - throwWhenClosed( - if (direction != ResultSet.FETCH_FORWARD) - throw new UJESSQLException( - UJESSQLErrorCode.NOSUPPORT_STATEMENT, - "only FETCH_FORWARD is supported." - ) - ) + throwWhenClosed(if (direction != ResultSet.FETCH_FORWARD) { + throw new UJESSQLException( + UJESSQLErrorCode.NOSUPPORT_STATEMENT, + "only FETCH_FORWARD is supported." + ) + }) override def getFetchDirection: Int = throwWhenClosed(ResultSet.FETCH_FORWARD) @@ -299,4 +280,33 @@ class UJESSQLStatement(private[jdbc] val ujesSQLConnection: UJESSQLConnection) throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_STATEMENT, "unwrap not supported") override def isWrapperFor(iface: Class[_]): Boolean = false + + /** + * log[0] error log[1] warn log[2] info log[3] all (info + warn + error) + * @return + */ + def getAllLog(): Array[String] = { + if (queryEnd && StringUtils.isNotBlank(logPath)) { + val openLogAction = + OpenLogAction.newBuilder().setLogPath(logPath).setProxyUser(ujesSQLConnection.user).build() + ujesSQLConnection.ujesClient.openLog(openLogAction).getLog + } else { + Array.empty[String] + } + } + + /** + * log[0] error log[1] warn log[2] info log[3] all (info + warn + error) + * @return + */ + def getIncrementalLog(): util.List[String] = { + if (null != jobExecuteResult && !queryEnd) { + val logObj = ujesSQLConnection.ujesClient.log(jobExecuteResult, logFromLen, logSize) + logFromLen = logObj.fromLine + logObj.getLog + } else { + new util.ArrayList[String] + } + } + } diff --git a/linkis-computation-governance/linkis-jdbc-driver/src/main/scala/org/apache/linkis/ujes/jdbc/UJESClientFactory.scala b/linkis-computation-governance/linkis-jdbc-driver/src/main/scala/org/apache/linkis/ujes/jdbc/UJESClientFactory.scala index 60f4d88af5..517f8b07ae 100644 --- a/linkis-computation-governance/linkis-jdbc-driver/src/main/scala/org/apache/linkis/ujes/jdbc/UJESClientFactory.scala +++ b/linkis-computation-governance/linkis-jdbc-driver/src/main/scala/org/apache/linkis/ujes/jdbc/UJESClientFactory.scala @@ -35,15 +35,27 @@ object UJESClientFactory extends Logging { def getUJESClient(props: Properties): UJESClient = { val host = props.getProperty(HOST) val port = props.getProperty(PORT) + val user = props.getProperty(USER) val serverUrl = if (StringUtils.isNotBlank(port)) s"http://$host:$port" else "http://" + host - if (ujesClients.containsKey(serverUrl)) ujesClients.get(serverUrl) - else - serverUrl.intern synchronized { - if (ujesClients.containsKey(serverUrl)) return ujesClients.get(serverUrl) + val uniqueKey = s"${serverUrl}_$user" + if (ujesClients.containsKey(uniqueKey)) { + logger.info("Clients with the same JDBC unique key({}) will get it directly", uniqueKey) + ujesClients.get(uniqueKey) + } else { + uniqueKey.intern synchronized { + if (ujesClients.containsKey(uniqueKey)) { + logger.info("Clients with the same JDBC unique key({}) will get it directly", uniqueKey) + return ujesClients.get(uniqueKey) + } + logger.info( + "The same Client does not exist for the JDBC unique key({}), a new Client will be created", + uniqueKey + ) val ujesClient = createUJESClient(serverUrl, props) - ujesClients.put(serverUrl, ujesClient) + ujesClients.put(uniqueKey, ujesClient) ujesClient } + } } private def createUJESClient(serverUrl: String, props: Properties): UJESClient = { diff --git a/linkis-computation-governance/linkis-jdbc-driver/src/main/scala/org/apache/linkis/ujes/jdbc/UJESSQLDatabaseMetaData.scala b/linkis-computation-governance/linkis-jdbc-driver/src/main/scala/org/apache/linkis/ujes/jdbc/UJESSQLDatabaseMetaData.scala index f2f0b9a106..ad856e81e0 100644 --- a/linkis-computation-governance/linkis-jdbc-driver/src/main/scala/org/apache/linkis/ujes/jdbc/UJESSQLDatabaseMetaData.scala +++ b/linkis-computation-governance/linkis-jdbc-driver/src/main/scala/org/apache/linkis/ujes/jdbc/UJESSQLDatabaseMetaData.scala @@ -29,7 +29,7 @@ import java.util import scala.collection.JavaConversions._ -class UJESSQLDatabaseMetaData(ujesSQLConnection: UJESSQLConnection) +class UJESSQLDatabaseMetaData(ujesSQLConnection: LinkisSQLConnection) extends DatabaseMetaData with Logging { override def allProceduresAreCallable(): Boolean = false @@ -358,7 +358,9 @@ class UJESSQLDatabaseMetaData(ujesSQLConnection: UJESSQLConnection) tableNamePattern: String, types: Array[String] ): ResultSet = { - val resultCatalog = if (StringUtils.isNotBlank(catalog)) { + val resultCatalog = if (StringUtils.isNotBlank(schemaPattern)) { + schemaPattern + } else if (StringUtils.isNotBlank(catalog)) { catalog } else { s"${getUserName}_ind" @@ -374,11 +376,16 @@ class UJESSQLDatabaseMetaData(ujesSQLConnection: UJESSQLConnection) if (table.get("isView").asInstanceOf[Boolean]) TableType.VIEW.name() else TableType.TABLE.name() val resultTable = new util.HashMap[String, String]() + val tableName = table.get("tableName").asInstanceOf[String] resultTable.put("catalog", resultCatalog) - resultTable.put("tableName", table.get("tableName").asInstanceOf[String]) + resultTable.put("tableName", tableName) resultTable.put("tableType", tableType) if (null == types || types.contains(tableType)) { - resultTables.add(resultTable) + if ( + StringUtils.isNotBlank(tableNamePattern) && tableNamePattern.equalsIgnoreCase(tableName) + ) { + resultTables.add(resultTable) + } } } val resultSet: LinkisMetaDataResultSet[util.Map[String, String]] = @@ -471,7 +478,9 @@ class UJESSQLDatabaseMetaData(ujesSQLConnection: UJESSQLConnection) tableNamePattern: String, columnNamePattern: String ): ResultSet = { - val resultCatalog = if (StringUtils.isNotBlank(catalog)) { + val resultCatalog = if (StringUtils.isNotBlank(schemaPattern)) { + schemaPattern + } else if (StringUtils.isNotBlank(catalog)) { catalog } else { s"${getUserName}_ind" diff --git a/linkis-computation-governance/linkis-jdbc-driver/src/main/scala/org/apache/linkis/ujes/jdbc/UJESSQLDriverMain.scala b/linkis-computation-governance/linkis-jdbc-driver/src/main/scala/org/apache/linkis/ujes/jdbc/UJESSQLDriverMain.scala index 783713cf40..9fb4c0f605 100644 --- a/linkis-computation-governance/linkis-jdbc-driver/src/main/scala/org/apache/linkis/ujes/jdbc/UJESSQLDriverMain.scala +++ b/linkis-computation-governance/linkis-jdbc-driver/src/main/scala/org/apache/linkis/ujes/jdbc/UJESSQLDriverMain.scala @@ -41,8 +41,10 @@ class UJESSQLDriverMain extends Driver with Logging { props.putAll(parseURL(url)) logger.info(s"input url:$url, properties:$properties") val ujesClient = UJESClientFactory.getUJESClient(props) - new UJESSQLConnection(ujesClient, props) - } else throw new UJESSQLException(UJESSQLErrorCode.BAD_URL, "bad url: " + url) + new LinkisSQLConnection(ujesClient, props) + } else { + null + } override def acceptsURL(url: String): Boolean = url.startsWith(URL_PREFIX) @@ -144,6 +146,8 @@ object UJESSQLDriverMain { val ENABLE_LOADBALANCER = UJESSQLDriver.ENABLE_LOADBALANCER val CREATOR = UJESSQLDriver.CREATOR + val TABLEAU = UJESSQLDriver.TABLEAU + val VARIABLE_HEADER = UJESSQLDriver.VARIABLE_HEADER def getConnectionParams( diff --git a/linkis-computation-governance/linkis-jdbc-driver/src/main/scala/org/apache/linkis/ujes/jdbc/UJESSQLResultSet.scala b/linkis-computation-governance/linkis-jdbc-driver/src/main/scala/org/apache/linkis/ujes/jdbc/UJESSQLResultSet.scala index b8cf1b23b1..97bfc576e3 100644 --- a/linkis-computation-governance/linkis-jdbc-driver/src/main/scala/org/apache/linkis/ujes/jdbc/UJESSQLResultSet.scala +++ b/linkis-computation-governance/linkis-jdbc-driver/src/main/scala/org/apache/linkis/ujes/jdbc/UJESSQLResultSet.scala @@ -42,7 +42,7 @@ import java.sql.{ Time, Timestamp } -import java.util.Calendar +import java.util.{Calendar, Locale} import org.joda.time.DateTimeZone import org.joda.time.format.{ @@ -54,7 +54,7 @@ import org.joda.time.format.{ class UJESSQLResultSet( resultSetList: Array[String], - ujesStatement: UJESSQLStatement, + ujesStatement: LinkisSQLStatement, maxRows: Int, fetchSize: Int ) extends ResultSet @@ -75,10 +75,10 @@ class UJESSQLResultSet( private val pageSize: Int = 5000 private var path: String = _ private var metaData: util.List[util.Map[String, String]] = _ - private val statement: UJESSQLStatement = ujesStatement + private val statement: LinkisSQLStatement = ujesStatement - private val connection: UJESSQLConnection = - ujesStatement.getConnection.asInstanceOf[UJESSQLConnection] + private val connection: LinkisSQLConnection = + ujesStatement.getConnection.asInstanceOf[LinkisSQLConnection] private var valueWasNull: Boolean = false private var warningChain: SQLWarning = _ @@ -96,6 +96,10 @@ class UJESSQLResultSet( .toFormatter .withOffsetParsed + private val STRING_TYPE = "string" + + private val NULL_VALUE = "NULL" + private def getResultSetPath(resultSetList: Array[String]): String = { if (resultSetList.length > 0) { resultSetList(resultSetList.length - 1) @@ -229,10 +233,14 @@ class UJESSQLResultSet( } private def evaluate(dataType: String, value: String): Any = { + if (value == null || value.equals("null") || value.equals("NULL") || value.equals("Null")) { - value + dataType.toLowerCase(Locale.getDefault) match { + case "string" | "char" | "varchar" | "nvarchar" => value + case _ => null + } } else { - dataType.toLowerCase match { + dataType.toLowerCase(Locale.getDefault) match { case null => throw new UJESSQLException(UJESSQLErrorCode.METADATA_EMPTY) case "string" => value.toString case "short" => value.toShort @@ -518,11 +526,7 @@ class UJESSQLResultSet( override def getObject(columnIndex: Int): Object = { val any = getColumnValue(columnIndex) - if (wasNull()) { - throw new UJESSQLException(UJESSQLErrorCode.RESULTSET_ROWERROR, "Type is null") - } else { - any.asInstanceOf[Object] - } + any.asInstanceOf[Object] } override def getObject(columnLabel: String): Object = { diff --git a/linkis-computation-governance/linkis-jdbc-driver/src/main/scala/org/apache/linkis/ujes/jdbc/UJESSQLTypeParser.scala b/linkis-computation-governance/linkis-jdbc-driver/src/main/scala/org/apache/linkis/ujes/jdbc/UJESSQLTypeParser.scala index ba4cd3878f..541064b2d5 100644 --- a/linkis-computation-governance/linkis-jdbc-driver/src/main/scala/org/apache/linkis/ujes/jdbc/UJESSQLTypeParser.scala +++ b/linkis-computation-governance/linkis-jdbc-driver/src/main/scala/org/apache/linkis/ujes/jdbc/UJESSQLTypeParser.scala @@ -18,11 +18,13 @@ package org.apache.linkis.ujes.jdbc import java.sql.{SQLException, Timestamp, Types} +import java.util.Locale object UJESSQLTypeParser { def parserFromName(typeName: String): Int = { - typeName.toLowerCase match { + val typeNameLowerCase = typeName.toLowerCase(Locale.getDefault()) + typeName.toLowerCase() match { case null => throw new UJESSQLException(UJESSQLErrorCode.METADATA_EMPTY) case "string" => Types.NVARCHAR case "short" => Types.SMALLINT @@ -43,7 +45,12 @@ object UJESSQLTypeParser { case "bigint" => Types.BIGINT case "array" => Types.ARRAY case "map" => Types.JAVA_OBJECT - case _ => throw new SQLException(s"parameter type error,Type:$typeName") + case _ => + if (typeNameLowerCase.startsWith("decimal")) { + Types.DECIMAL + } else { + Types.NVARCHAR + } } } diff --git a/linkis-computation-governance/linkis-jdbc-driver/src/main/scala/org/apache/linkis/ujes/jdbc/hook/JDBCDriverPreExecutionHook.scala b/linkis-computation-governance/linkis-jdbc-driver/src/main/scala/org/apache/linkis/ujes/jdbc/hook/JDBCDriverPreExecutionHook.scala index 038ff38712..ed8936cc77 100644 --- a/linkis-computation-governance/linkis-jdbc-driver/src/main/scala/org/apache/linkis/ujes/jdbc/hook/JDBCDriverPreExecutionHook.scala +++ b/linkis-computation-governance/linkis-jdbc-driver/src/main/scala/org/apache/linkis/ujes/jdbc/hook/JDBCDriverPreExecutionHook.scala @@ -24,7 +24,7 @@ import scala.collection.mutable.ArrayBuffer trait JDBCDriverPreExecutionHook { - def callPreExecutionHook(sql: String): String + def callPreExecutionHook(sql: String, skip: Boolean): String } @@ -51,5 +51,5 @@ object JDBCDriverPreExecutionHook extends Logging { hooks.toArray } - def getPreExecutionHooks = preExecutionHooks + def getPreExecutionHooks: Array[JDBCDriverPreExecutionHook] = preExecutionHooks } diff --git a/linkis-computation-governance/linkis-jdbc-driver/src/main/scala/org/apache/linkis/ujes/jdbc/hook/impl/NoLimitExecutionHook.scala b/linkis-computation-governance/linkis-jdbc-driver/src/main/scala/org/apache/linkis/ujes/jdbc/hook/impl/NoLimitExecutionHook.scala index 18f94c0fde..103a544772 100644 --- a/linkis-computation-governance/linkis-jdbc-driver/src/main/scala/org/apache/linkis/ujes/jdbc/hook/impl/NoLimitExecutionHook.scala +++ b/linkis-computation-governance/linkis-jdbc-driver/src/main/scala/org/apache/linkis/ujes/jdbc/hook/impl/NoLimitExecutionHook.scala @@ -20,10 +20,12 @@ package org.apache.linkis.ujes.jdbc.hook.impl import org.apache.linkis.ujes.jdbc.UJESSQLDriverMain import org.apache.linkis.ujes.jdbc.hook.JDBCDriverPreExecutionHook +import java.util.Locale + class NoLimitExecutionHook extends JDBCDriverPreExecutionHook { - override def callPreExecutionHook(sql: String): String = { - if (UJESSQLDriverMain.LIMIT_ENABLED.toLowerCase.equals("false")) { + override def callPreExecutionHook(sql: String, skip: Boolean): String = { + if (UJESSQLDriverMain.LIMIT_ENABLED.equalsIgnoreCase("false")) { var noLimitSql = "--set ide.engine.no.limit.allow=true\n" + sql val lowerCaseLimitSql = noLimitSql.toLowerCase() if (lowerCaseLimitSql.contains("limit ") && lowerCaseLimitSql.contains("tableausql")) { diff --git a/linkis-computation-governance/linkis-jdbc-driver/src/main/scala/org/apache/linkis/ujes/jdbc/hook/impl/TableauPreExecutionHook.scala b/linkis-computation-governance/linkis-jdbc-driver/src/main/scala/org/apache/linkis/ujes/jdbc/hook/impl/TableauPreExecutionHook.scala index 63c4f53c9f..ec177b4de7 100644 --- a/linkis-computation-governance/linkis-jdbc-driver/src/main/scala/org/apache/linkis/ujes/jdbc/hook/impl/TableauPreExecutionHook.scala +++ b/linkis-computation-governance/linkis-jdbc-driver/src/main/scala/org/apache/linkis/ujes/jdbc/hook/impl/TableauPreExecutionHook.scala @@ -21,7 +21,10 @@ import org.apache.linkis.ujes.jdbc.hook.JDBCDriverPreExecutionHook class TableauPreExecutionHook extends JDBCDriverPreExecutionHook { - override def callPreExecutionHook(sql: String): String = { + override def callPreExecutionHook(sql: String, skip: Boolean): String = { + if (skip) { + return sql + } if ( sql.contains("CREATE INDEX") || sql .contains("CREATE TABLE") || sql.contains("INSERT INTO") || sql.contains("DROP TABLE") diff --git a/linkis-computation-governance/linkis-jdbc-driver/src/test/java/org/apache/linkis/ujes/jdbc/CreateConnection.java b/linkis-computation-governance/linkis-jdbc-driver/src/test/java/org/apache/linkis/ujes/jdbc/CreateConnection.java index 51d9e9953a..ca2c1e35fd 100644 --- a/linkis-computation-governance/linkis-jdbc-driver/src/test/java/org/apache/linkis/ujes/jdbc/CreateConnection.java +++ b/linkis-computation-governance/linkis-jdbc-driver/src/test/java/org/apache/linkis/ujes/jdbc/CreateConnection.java @@ -27,12 +27,12 @@ public class CreateConnection { - private static UJESSQLConnection conn; + private static LinkisSQLConnection conn; - public static UJESSQLConnection getConnection() throws ClassNotFoundException, SQLException { + public static LinkisSQLConnection getConnection() throws ClassNotFoundException, SQLException { Class.forName("org.apache.linkis.ujes.jdbc.UJESSQLDriver"); conn = - (UJESSQLConnection) + (LinkisSQLConnection) DriverManager.getConnection("jdbc:linkis://hostname:port", "username", "password"); return conn; } diff --git a/linkis-computation-governance/linkis-jdbc-driver/src/test/java/org/apache/linkis/ujes/jdbc/JDBCSpiTest.java b/linkis-computation-governance/linkis-jdbc-driver/src/test/java/org/apache/linkis/ujes/jdbc/JDBCSpiTest.java index 427a6b5fc4..3e76bb2856 100644 --- a/linkis-computation-governance/linkis-jdbc-driver/src/test/java/org/apache/linkis/ujes/jdbc/JDBCSpiTest.java +++ b/linkis-computation-governance/linkis-jdbc-driver/src/test/java/org/apache/linkis/ujes/jdbc/JDBCSpiTest.java @@ -29,21 +29,21 @@ * */ public class JDBCSpiTest { - private static UJESSQLConnection conn; + private static LinkisSQLConnection conn; - public static UJESSQLConnection getConnection() throws ClassNotFoundException, SQLException { + public static LinkisSQLConnection getConnection() throws ClassNotFoundException, SQLException { Class.forName("org.apache.linkis.ujes.jdbc.UJESSQLDriver"); conn = - (UJESSQLConnection) - DriverManager.getConnection("jdbc:linkis://hostname:port", "root", "123456"); + (LinkisSQLConnection) + DriverManager.getConnection("jdbc:linkis://127.0.0.1:9001", "root", "123456"); return conn; } @Test public void spiTest() { try { - UJESSQLConnection conn = - (UJESSQLConnection) + LinkisSQLConnection conn = + (LinkisSQLConnection) DriverManager.getConnection("jdbc:linkis://hostname:port", "username", "password"); Assertions.assertNotNull(conn); } catch (SQLException e) { diff --git a/linkis-computation-governance/linkis-jdbc-driver/src/test/java/org/apache/linkis/ujes/jdbc/UJESSQLStatementTest.java b/linkis-computation-governance/linkis-jdbc-driver/src/test/java/org/apache/linkis/ujes/jdbc/LinkisSQLStatementTest.java similarity index 95% rename from linkis-computation-governance/linkis-jdbc-driver/src/test/java/org/apache/linkis/ujes/jdbc/UJESSQLStatementTest.java rename to linkis-computation-governance/linkis-jdbc-driver/src/test/java/org/apache/linkis/ujes/jdbc/LinkisSQLStatementTest.java index e674dd5b23..3ebd21ae70 100644 --- a/linkis-computation-governance/linkis-jdbc-driver/src/test/java/org/apache/linkis/ujes/jdbc/UJESSQLStatementTest.java +++ b/linkis-computation-governance/linkis-jdbc-driver/src/test/java/org/apache/linkis/ujes/jdbc/LinkisSQLStatementTest.java @@ -33,9 +33,9 @@ * if you want to test this module,you must rewrite default parameters and SQL we used for local test * */ -public class UJESSQLStatementTest { - private static UJESSQLConnection conn; - private static UJESSQLStatement statement; +public class LinkisSQLStatementTest { + private static LinkisSQLConnection conn; + private static LinkisSQLStatement statement; private static int maxRows; private static int queryTimeout; private static String sql; @@ -48,7 +48,7 @@ public class UJESSQLStatementTest { public static void createConnection() { try { conn = CreateConnection.getConnection(); - statement = (UJESSQLStatement) conn.createStatement(); + statement = (LinkisSQLStatement) conn.createStatement(); } catch (ClassNotFoundException e) { e.printStackTrace(); } catch (SQLException e) { diff --git a/linkis-computation-governance/linkis-jdbc-driver/src/test/java/org/apache/linkis/ujes/jdbc/UJESSQLDatabaseMetaDataTest.java b/linkis-computation-governance/linkis-jdbc-driver/src/test/java/org/apache/linkis/ujes/jdbc/UJESSQLDatabaseMetaDataTest.java index 0dab63b3ff..e7c6ade996 100644 --- a/linkis-computation-governance/linkis-jdbc-driver/src/test/java/org/apache/linkis/ujes/jdbc/UJESSQLDatabaseMetaDataTest.java +++ b/linkis-computation-governance/linkis-jdbc-driver/src/test/java/org/apache/linkis/ujes/jdbc/UJESSQLDatabaseMetaDataTest.java @@ -34,7 +34,7 @@ import static org.junit.jupiter.api.Assertions.assertTrue; public class UJESSQLDatabaseMetaDataTest { - private static UJESSQLConnection conn; + private static LinkisSQLConnection conn; private static UJESSQLDatabaseMetaData dbmd; @BeforeAll diff --git a/linkis-computation-governance/linkis-jdbc-driver/src/test/java/org/apache/linkis/ujes/jdbc/UJESSQLPreparedStatementTest.java b/linkis-computation-governance/linkis-jdbc-driver/src/test/java/org/apache/linkis/ujes/jdbc/UJESSQLPreparedStatementTest.java index 54bc3d4538..da431c82e7 100644 --- a/linkis-computation-governance/linkis-jdbc-driver/src/test/java/org/apache/linkis/ujes/jdbc/UJESSQLPreparedStatementTest.java +++ b/linkis-computation-governance/linkis-jdbc-driver/src/test/java/org/apache/linkis/ujes/jdbc/UJESSQLPreparedStatementTest.java @@ -35,8 +35,8 @@ * */ public class UJESSQLPreparedStatementTest { - private static UJESSQLConnection conn; - private UJESSQLPreparedStatement preStatement; + private static LinkisSQLConnection conn; + private LinkisSQLPreparedStatement preStatement; @BeforeAll public static void getConnection() { diff --git a/linkis-computation-governance/linkis-jdbc-driver/src/test/java/org/apache/linkis/ujes/jdbc/UJESSQLResultSetTest.java b/linkis-computation-governance/linkis-jdbc-driver/src/test/java/org/apache/linkis/ujes/jdbc/UJESSQLResultSetTest.java index 0a545d8012..a8f0a179d0 100644 --- a/linkis-computation-governance/linkis-jdbc-driver/src/test/java/org/apache/linkis/ujes/jdbc/UJESSQLResultSetTest.java +++ b/linkis-computation-governance/linkis-jdbc-driver/src/test/java/org/apache/linkis/ujes/jdbc/UJESSQLResultSetTest.java @@ -33,8 +33,8 @@ public class UJESSQLResultSetTest { - private static UJESSQLConnection conn; - private UJESSQLPreparedStatement preStatement; + private static LinkisSQLConnection conn; + private LinkisSQLPreparedStatement preStatement; private UJESSQLResultSet resultSet; private UJESSQLResultSetMetaData metaData; diff --git a/linkis-spring-cloud-services/linkis-service-gateway/linkis-gateway-httpclient-support/src/main/scala/org/apache/linkis/httpclient/dws/response/DWSResult.scala b/linkis-spring-cloud-services/linkis-service-gateway/linkis-gateway-httpclient-support/src/main/scala/org/apache/linkis/httpclient/dws/response/DWSResult.scala index bada2315cb..3f00f61f5f 100644 --- a/linkis-spring-cloud-services/linkis-service-gateway/linkis-gateway-httpclient-support/src/main/scala/org/apache/linkis/httpclient/dws/response/DWSResult.scala +++ b/linkis-spring-cloud-services/linkis-service-gateway/linkis-gateway-httpclient-support/src/main/scala/org/apache/linkis/httpclient/dws/response/DWSResult.scala @@ -82,7 +82,9 @@ trait DWSResult extends Logging with HttpResult { this.url = url this.contentType = contentType } { case e: Exception => - logger.error(e.getMessage()) + throw new HttpClientResultException( + s"URL $url request failed! ResponseBody is $responseBody. ${e.getMessage}" + ) } } From d2bffc25861d666a7eae24e31a9834d3147b58be Mon Sep 17 00:00:00 2001 From: Casion Date: Tue, 9 May 2023 14:19:29 +0800 Subject: [PATCH 102/261] fix eclist api:filter after group by (#151) --- .../mapper/common/ECResourceRecordMapper.xml | 39 ++++++++++--------- 1 file changed, 20 insertions(+), 19 deletions(-) diff --git a/linkis-computation-governance/linkis-manager/linkis-manager-persistence/src/main/resources/mapper/common/ECResourceRecordMapper.xml b/linkis-computation-governance/linkis-manager/linkis-manager-persistence/src/main/resources/mapper/common/ECResourceRecordMapper.xml index 9c9eb2cc63..6806b7e8d4 100644 --- a/linkis-computation-governance/linkis-manager/linkis-manager-persistence/src/main/resources/mapper/common/ECResourceRecordMapper.xml +++ b/linkis-computation-governance/linkis-manager/linkis-manager-persistence/src/main/resources/mapper/common/ECResourceRecordMapper.xml @@ -83,25 +83,26 @@ \ No newline at end of file From b9c2fd898c27c8aeddca3136355529d0823943b3 Mon Sep 17 00:00:00 2001 From: ahaoyao <129247228+ahaoyao@users.noreply.github.com> Date: Wed, 10 May 2023 14:42:52 +0800 Subject: [PATCH 103/261] Optimization of token exception scenario description (#152) * Optimization of token exception scenario description --- ...inkisGwAuthenticationErrorCodeSummary.java | 3 +- .../service/CachedTokenService.scala | 34 ++++++++++--------- 2 files changed, 20 insertions(+), 17 deletions(-) diff --git a/linkis-spring-cloud-services/linkis-service-gateway/linkis-gateway-authentication/src/main/java/org/apache/linkis/gateway/authentication/errorcode/LinkisGwAuthenticationErrorCodeSummary.java b/linkis-spring-cloud-services/linkis-service-gateway/linkis-gateway-authentication/src/main/java/org/apache/linkis/gateway/authentication/errorcode/LinkisGwAuthenticationErrorCodeSummary.java index bfdba4a28e..236fb892b9 100644 --- a/linkis-spring-cloud-services/linkis-service-gateway/linkis-gateway-authentication/src/main/java/org/apache/linkis/gateway/authentication/errorcode/LinkisGwAuthenticationErrorCodeSummary.java +++ b/linkis-spring-cloud-services/linkis-service-gateway/linkis-gateway-authentication/src/main/java/org/apache/linkis/gateway/authentication/errorcode/LinkisGwAuthenticationErrorCodeSummary.java @@ -28,7 +28,8 @@ public enum LinkisGwAuthenticationErrorCodeSummary implements LinkisErrorCode { ILLEGAL_HOST(15203, "Illegal Host for Token(Token非法主机)!"), INVALID_TOKEN(15204, "Invalid Token(令牌无效)"), TOKEN_IS_NULL(15205, "token is null(token 令牌为空)!"), - FAILED_TO_BAD_SQLGRAMMAR(15206, "Failed to query token{0} data(Token:{0} 数据查询失败), Caused by:{1}"), + FAILED_TO_BAD_SQLGRAMMAR( + 15206, "Failed to query token:{0} data(Token:{0} 数据查询失败), Caused by:{1}"), NOT_EXIST_DB(15207, "Token:{0} does not exist in the table(Token:{0} 表中不存在)!, Caused by:{1}"); /** (errorCode)错误码 */ diff --git a/linkis-spring-cloud-services/linkis-service-gateway/linkis-gateway-authentication/src/main/scala/org/apache/linkis/gateway/authentication/service/CachedTokenService.scala b/linkis-spring-cloud-services/linkis-service-gateway/linkis-gateway-authentication/src/main/scala/org/apache/linkis/gateway/authentication/service/CachedTokenService.scala index b9b91d23e6..85adce953e 100644 --- a/linkis-spring-cloud-services/linkis-service-gateway/linkis-gateway-authentication/src/main/scala/org/apache/linkis/gateway/authentication/service/CachedTokenService.scala +++ b/linkis-spring-cloud-services/linkis-service-gateway/linkis-gateway-authentication/src/main/scala/org/apache/linkis/gateway/authentication/service/CachedTokenService.scala @@ -23,6 +23,7 @@ import org.apache.linkis.gateway.authentication.bo.impl.TokenImpl import org.apache.linkis.gateway.authentication.conf.TokenConfiguration import org.apache.linkis.gateway.authentication.dao.TokenDao import org.apache.linkis.gateway.authentication.entity.TokenEntity +import org.apache.linkis.gateway.authentication.errorcode.LinkisGwAuthenticationErrorCodeSummary import org.apache.linkis.gateway.authentication.errorcode.LinkisGwAuthenticationErrorCodeSummary._ import org.apache.linkis.gateway.authentication.exception.{ TokenAuthException, @@ -112,30 +113,31 @@ class CachedTokenService extends TokenService { case x: ExecutionException => x.getCause match { case e: TokenNotExistException => - throw new TokenAuthException( - NOT_EXIST_DB.getErrorCode, - MessageFormat.format(NOT_EXIST_DB.getErrorDesc, tokenName, e.getMessage) - ) + throwTokenAuthException(NOT_EXIST_DB, tokenName, e) case e => - throw new TokenAuthException( - FAILED_TO_LOAD_TOKEN.getErrorCode, - MessageFormat.format(FAILED_TO_LOAD_TOKEN.getErrorDesc, tokenName, e.getMessage) - ) + throwTokenAuthException(FAILED_TO_LOAD_TOKEN, tokenName, e) } case e: UncheckedExecutionException => - throw new TokenAuthException( - FAILED_TO_BAD_SQLGRAMMAR.getErrorCode, - MessageFormat.format(FAILED_TO_BAD_SQLGRAMMAR.getErrorDesc, tokenName, e.getMessage) - ) + throwTokenAuthException(FAILED_TO_BAD_SQLGRAMMAR, tokenName, e) case e => - throw new TokenAuthException( - FAILED_TO_LOAD_TOKEN.getErrorCode, - MessageFormat.format(FAILED_TO_LOAD_TOKEN.getErrorDesc, tokenName, e.getMessage) - ) + throwTokenAuthException(FAILED_TO_LOAD_TOKEN, tokenName, e) } ) } + private def throwTokenAuthException( + gwAuthenticationErrorCodeSummary: LinkisGwAuthenticationErrorCodeSummary, + tokenName: String, + e: Throwable + ) = { + val exception = new TokenAuthException( + gwAuthenticationErrorCodeSummary.getErrorCode, + MessageFormat.format(gwAuthenticationErrorCodeSummary.getErrorDesc, tokenName, e.getMessage) + ) + exception.initCause(e) + throw exception + } + private def isTokenAcceptableWithUser(token: Token, userName: String): Boolean = { token != null && !token.isStale() && token.isUserLegal(userName) } From 97a38bb482fb17563057dea16626234bb1532ffb Mon Sep 17 00:00:00 2001 From: huangKai-2323 <62878639+huangKai-2323@users.noreply.github.com> Date: Wed, 10 May 2023 15:06:17 +0800 Subject: [PATCH 104/261] ec list add unlock time---code_view (#153) * fix class no found * fix NEP * fix NEP * code format * code review --------- Co-authored-by: casionone --- .../manager/am/restful/ECResourceInfoRestfulApi.java | 2 +- .../manager/am/service/ECResourceInfoService.java | 4 ++-- .../am/service/impl/ECResourceInfoServiceImpl.java | 10 +++++----- .../apache/linkis/manager/dao/NodeManagerMapper.java | 2 +- .../main/resources/mapper/common/NodeManagerMapper.xml | 4 ++-- 5 files changed, 11 insertions(+), 11 deletions(-) diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/restful/ECResourceInfoRestfulApi.java b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/restful/ECResourceInfoRestfulApi.java index 65e855061d..60734a7056 100644 --- a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/restful/ECResourceInfoRestfulApi.java +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/restful/ECResourceInfoRestfulApi.java @@ -189,7 +189,7 @@ public Message queryEcrHistory( @ApiImplicitParam(name = "engineTypes", dataType = "Array", value = "engine type"), @ApiImplicitParam(name = "statuss", dataType = "Array", value = "statuss"), @ApiImplicitParam(name = "queueName", dataType = "String", value = "queueName"), - @ApiImplicitParam(name = "ecInstances", dataType = "String", value = "ecInstances"), + @ApiImplicitParam(name = "ecInstances", dataType = "Array", value = "ecInstances"), }) @RequestMapping(path = "/ecList", method = RequestMethod.POST) public Message queryEcList(HttpServletRequest req, @RequestBody JsonNode jsonNode) { diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/service/ECResourceInfoService.java b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/service/ECResourceInfoService.java index b7ac14866e..fd9b384359 100644 --- a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/service/ECResourceInfoService.java +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/service/ECResourceInfoService.java @@ -41,7 +41,7 @@ List getECResourceInfoRecordList( * @param engineTypeList engineconn type list * @param statusStrList engineconn status string list * @param queueName - * @param instanceNameList + * @param ecInstancesList * @return */ List> getECResourceInfoList( @@ -49,5 +49,5 @@ List> getECResourceInfoList( List engineTypeList, List statusStrList, String queueName, - List instanceNameList); + List ecInstancesList); } diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/service/impl/ECResourceInfoServiceImpl.java b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/service/impl/ECResourceInfoServiceImpl.java index 2c00a3f84f..953781586f 100644 --- a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/service/impl/ECResourceInfoServiceImpl.java +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/service/impl/ECResourceInfoServiceImpl.java @@ -108,7 +108,7 @@ public List> getECResourceInfoList( List engineTypeList, List statusStrList, String queueName, - List instanceNameList) { + List ecInstancesList) { List> resultList = new ArrayList<>(); @@ -120,7 +120,7 @@ public List> getECResourceInfoList( // get engine conn info list filter by creator user list /instance status list List ecNodesInfo = - nodeManagerMapper.getEMNodeInfoList(creatorUserList, statusIntList, instanceNameList); + nodeManagerMapper.getEMNodeInfoList(creatorUserList, statusIntList, ecInstancesList); // map k:v---> instanceName：PersistencerEcNodeInfo Map persistencerEcNodeInfoMap = @@ -150,8 +150,8 @@ public List> getECResourceInfoList( json.writeValueAsString(ecNodeinfo), new TypeReference>() {}); - Integer intStatus = ecNodeinfo.getInstanceStatus(); - item.put("instanceStatus", NodeStatus.values()[intStatus].name()); + Integer instanceStatus = ecNodeinfo.getInstanceStatus(); + item.put("instanceStatus", NodeStatus.values()[instanceStatus].name()); String usedResourceStr = latestRecord.getUsedResource(); /* @@ -160,7 +160,7 @@ public List> getECResourceInfoList( {"driver":{"instance":1,"memory":"2.0 GB","cpu":1} } */ long lastUnlockTimestamp = 0L; - if (NodeStatus.values()[intStatus].name().equals(NodeStatus.Unlock.name())) { + if (NodeStatus.values()[instanceStatus].name().equals(NodeStatus.Unlock.name())) { String heartbeatMsg = ecNodeinfo.getHeartbeatMsg(); Map heartbeatMap = new HashMap<>(); if (StringUtils.isNotBlank(heartbeatMsg)) { diff --git a/linkis-computation-governance/linkis-manager/linkis-manager-persistence/src/main/java/org/apache/linkis/manager/dao/NodeManagerMapper.java b/linkis-computation-governance/linkis-manager/linkis-manager-persistence/src/main/java/org/apache/linkis/manager/dao/NodeManagerMapper.java index 910102895c..44dd771b4c 100644 --- a/linkis-computation-governance/linkis-manager/linkis-manager-persistence/src/main/java/org/apache/linkis/manager/dao/NodeManagerMapper.java +++ b/linkis-computation-governance/linkis-manager/linkis-manager-persistence/src/main/java/org/apache/linkis/manager/dao/NodeManagerMapper.java @@ -79,5 +79,5 @@ void updateNodeLabelRelation( List getEMNodeInfoList( @Param("creatorUsers") List creatorUsers, @Param("statuss") List statuss, - @Param("instanceName") List instanceNameList); + @Param("ecInstancesList") List ecInstancesList); } diff --git a/linkis-computation-governance/linkis-manager/linkis-manager-persistence/src/main/resources/mapper/common/NodeManagerMapper.xml b/linkis-computation-governance/linkis-manager/linkis-manager-persistence/src/main/resources/mapper/common/NodeManagerMapper.xml index 070ad68029..09d923c007 100644 --- a/linkis-computation-governance/linkis-manager/linkis-manager-persistence/src/main/resources/mapper/common/NodeManagerMapper.xml +++ b/linkis-computation-governance/linkis-manager/linkis-manager-persistence/src/main/resources/mapper/common/NodeManagerMapper.xml @@ -133,9 +133,9 @@ #{i} - + and t.instance in - + #{i} From 141a04c0d5d11910c5bd33540e95952a229d9889 Mon Sep 17 00:00:00 2001 From: huangKai-2323 <62878639+huangKai-2323@users.noreply.github.com> Date: Wed, 10 May 2023 15:07:51 +0800 Subject: [PATCH 105/261] optimize udf permissions (#150) * Code optimization * add udf function createtime && updatetime column * add udf updatetime sql --- linkis-dist/package/db/linkis_ddl.sql | 3 +- linkis-dist/package/db/module/linkis_udf.sql | 2 + .../upgrade/1.4.0_schema/mysql/linkis_ddl.sql | 2 + .../server/domain/UdfTreeEntity.java | 12 +++ .../server/restful/UdfManagerRestfulApi.java | 7 +- .../server/restful/UdfTreeRestfulApi.java | 16 +--- .../server/utils/UdfTreeUtils.java | 93 +++++++++++++++++++ .../src/test/resources/create.sql | 13 +-- 8 files changed, 126 insertions(+), 22 deletions(-) create mode 100644 linkis-public-enhancements/linkis-basedata-manager/src/main/java/org/apache/linkis/basedatamanager/server/utils/UdfTreeUtils.java diff --git a/linkis-dist/package/db/linkis_ddl.sql b/linkis-dist/package/db/linkis_ddl.sql index 10e1ada0a6..b4df979f84 100644 --- a/linkis-dist/package/db/linkis_ddl.sql +++ b/linkis-dist/package/db/linkis_ddl.sql @@ -196,7 +196,8 @@ CREATE TABLE `linkis_ps_udf_tree` ( `create_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, `update_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, `category` varchar(50) DEFAULT NULL COMMENT 'Used to distinguish between udf and function', - PRIMARY KEY (`id`) + PRIMARY KEY (`id`), + UNIQUE KEY `uniq_name_uname_category` (`name`, `user_name`, `category`) ) ENGINE=InnoDB DEFAULT CHARSET=utf8; diff --git a/linkis-dist/package/db/module/linkis_udf.sql b/linkis-dist/package/db/module/linkis_udf.sql index 999793b1cc..783591d041 100644 --- a/linkis-dist/package/db/module/linkis_udf.sql +++ b/linkis-dist/package/db/module/linkis_udf.sql @@ -25,6 +25,8 @@ DROP TABLE IF EXISTS `linkis_ps_udf_manager`; CREATE TABLE `linkis_ps_udf_manager` ( `id` bigint(20) NOT NULL AUTO_INCREMENT, `user_name` varchar(20) DEFAULT NULL, + `update_time` datetime DEFAULT CURRENT_TIMESTAMP, + `create_time` datetime DEFAULT CURRENT_TIMESTAMP, PRIMARY KEY (`id`) ) ENGINE=InnoDB DEFAULT CHARSET=utf8; diff --git a/linkis-dist/package/db/upgrade/1.4.0_schema/mysql/linkis_ddl.sql b/linkis-dist/package/db/upgrade/1.4.0_schema/mysql/linkis_ddl.sql index f431faee65..8de7cf7632 100644 --- a/linkis-dist/package/db/upgrade/1.4.0_schema/mysql/linkis_ddl.sql +++ b/linkis-dist/package/db/upgrade/1.4.0_schema/mysql/linkis_ddl.sql @@ -29,3 +29,5 @@ ALTER TABLE `linkis_ps_udf_user_load` ADD COLUMN `update_time` datetime NOT NUL ALTER TABLE `linkis_ps_udf_version` ADD COLUMN `update_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'update_time'; + +ALTER TABLE `linkis_ps_udf_tree` ADD CONSTRAINT `uniq_name_uname_category` UNIQUE (`name`, `user_name`, `category`); \ No newline at end of file diff --git a/linkis-public-enhancements/linkis-basedata-manager/src/main/java/org/apache/linkis/basedatamanager/server/domain/UdfTreeEntity.java b/linkis-public-enhancements/linkis-basedata-manager/src/main/java/org/apache/linkis/basedatamanager/server/domain/UdfTreeEntity.java index 9d2a51d471..32615f095f 100644 --- a/linkis-public-enhancements/linkis-basedata-manager/src/main/java/org/apache/linkis/basedatamanager/server/domain/UdfTreeEntity.java +++ b/linkis-public-enhancements/linkis-basedata-manager/src/main/java/org/apache/linkis/basedatamanager/server/domain/UdfTreeEntity.java @@ -19,6 +19,7 @@ import java.io.Serializable; import java.util.Date; +import java.util.List; import com.baomidou.mybatisplus.annotation.IdType; import com.baomidou.mybatisplus.annotation.TableField; @@ -58,6 +59,9 @@ public class UdfTreeEntity implements Serializable { @TableField(exist = false) private static final long serialVersionUID = 1L; + @TableField(exist = false) + private List childrenList; + /** */ public Long getId() { return id; @@ -138,6 +142,14 @@ public void setCategory(String category) { this.category = category; } + public List getChildrenList() { + return childrenList; + } + + public void setChildrenList(List childrenList) { + this.childrenList = childrenList; + } + @Override public boolean equals(Object that) { if (this == that) { diff --git a/linkis-public-enhancements/linkis-basedata-manager/src/main/java/org/apache/linkis/basedatamanager/server/restful/UdfManagerRestfulApi.java b/linkis-public-enhancements/linkis-basedata-manager/src/main/java/org/apache/linkis/basedatamanager/server/restful/UdfManagerRestfulApi.java index f048b12f4b..e56e1bda48 100644 --- a/linkis-public-enhancements/linkis-basedata-manager/src/main/java/org/apache/linkis/basedatamanager/server/restful/UdfManagerRestfulApi.java +++ b/linkis-public-enhancements/linkis-basedata-manager/src/main/java/org/apache/linkis/basedatamanager/server/restful/UdfManagerRestfulApi.java @@ -96,7 +96,7 @@ public Message add(HttpServletRequest request, @RequestBody UdfManagerEntity udf boolean result = udfManagerService.save(udfManagerEntity); return Message.ok("").data("result", result); } else { - return Message.error("The username already exists,Please add again!"); + return Message.error("The " + udfManager.getUserName() + " already exists,Please add again!"); } } @@ -134,11 +134,12 @@ public Message update( new QueryWrapper<>(udfManagerEntity).eq("user_name", udfManagerEntity.getUserName()); UdfManagerEntity udfManager = udfManagerService.getOne(queryWrapper); if (udfManager == null) { - udfManager.setUpdateTime(new Date()); + udfManagerEntity.setUpdateTime(new Date()); boolean result = udfManagerService.updateById(udfManagerEntity); return Message.ok("").data("result", result); } else { - return Message.error("The username already exists,Please update again!"); + return Message.error( + "The " + udfManager.getUserName() + " already exists,Please update again!"); } } } diff --git a/linkis-public-enhancements/linkis-basedata-manager/src/main/java/org/apache/linkis/basedatamanager/server/restful/UdfTreeRestfulApi.java b/linkis-public-enhancements/linkis-basedata-manager/src/main/java/org/apache/linkis/basedatamanager/server/restful/UdfTreeRestfulApi.java index c9fd2bf985..7b8f434236 100644 --- a/linkis-public-enhancements/linkis-basedata-manager/src/main/java/org/apache/linkis/basedatamanager/server/restful/UdfTreeRestfulApi.java +++ b/linkis-public-enhancements/linkis-basedata-manager/src/main/java/org/apache/linkis/basedatamanager/server/restful/UdfTreeRestfulApi.java @@ -17,11 +17,11 @@ package org.apache.linkis.basedatamanager.server.restful; -import org.apache.linkis.basedatamanager.server.conf.UdfTreeConf; import org.apache.linkis.basedatamanager.server.domain.UdfBaseInfoEntity; import org.apache.linkis.basedatamanager.server.domain.UdfTreeEntity; import org.apache.linkis.basedatamanager.server.service.UdfBaseInfoService; import org.apache.linkis.basedatamanager.server.service.UdfTreeService; +import org.apache.linkis.basedatamanager.server.utils.UdfTreeUtils; import org.apache.linkis.common.conf.Configuration; import org.apache.linkis.server.Message; import org.apache.linkis.server.utils.ModuleUserUtils; @@ -39,7 +39,6 @@ import javax.servlet.http.HttpServletRequest; import java.util.ArrayList; -import java.util.Arrays; import java.util.List; import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper; @@ -83,22 +82,15 @@ public Message all(HttpServletRequest request, String searchName, String categor ModuleUserUtils.getOperationUser( request, "Query all data of UDF Tree,search name:" + searchName); List udfTreeEntityList = new ArrayList<>(); - UdfTreeEntity entity = new UdfTreeEntity(); - entity.setCategory(category); - QueryWrapper querySysWrapper = - new QueryWrapper<>(entity) - .eq("category", entity.getCategory()) - .in( - "user_name", - Arrays.asList(UdfTreeConf.UDF_FUN_SYSTEM_CATEGORY.getValue().split(","))); - udfTreeEntityList = udfTreeService.list(querySysWrapper); if (StringUtils.isNotBlank(searchName) && StringUtils.isNotBlank(category)) { + UdfTreeEntity entity = new UdfTreeEntity(); + entity.setCategory(category); entity.setUserName(searchName); QueryWrapper queryWrapper = new QueryWrapper<>(entity) .eq("user_name", entity.getUserName()) .eq("category", entity.getCategory()); - udfTreeEntityList.addAll(udfTreeService.list(queryWrapper)); + udfTreeEntityList = new UdfTreeUtils(udfTreeService.list(queryWrapper)).buildTree(); } return Message.ok("").data("list", udfTreeEntityList); } diff --git a/linkis-public-enhancements/linkis-basedata-manager/src/main/java/org/apache/linkis/basedatamanager/server/utils/UdfTreeUtils.java b/linkis-public-enhancements/linkis-basedata-manager/src/main/java/org/apache/linkis/basedatamanager/server/utils/UdfTreeUtils.java new file mode 100644 index 0000000000..f7f2b19fca --- /dev/null +++ b/linkis-public-enhancements/linkis-basedata-manager/src/main/java/org/apache/linkis/basedatamanager/server/utils/UdfTreeUtils.java @@ -0,0 +1,93 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.basedatamanager.server.utils; + +import org.apache.linkis.basedatamanager.server.domain.UdfTreeEntity; + +import java.util.ArrayList; +import java.util.List; + +public class UdfTreeUtils { + + /** Build tree structure */ + public List udfTreeList = new ArrayList<>(); + + /** Construction method */ + public UdfTreeUtils(List udfTreeList) { + this.udfTreeList = udfTreeList; + } + + /** + * Obtain all root nodes (top-level nodes) that need to be built + * + * @return All Root Node List Collection + */ + public List getRootNode() { + // Save all root nodes (data for all root nodes) + List rootudfTreeList = new ArrayList<>(); + // UdfTreeEntity: Each piece of data (node) found in the query + for (UdfTreeEntity UdfTreeEntity : udfTreeList) { + // Determine whether the current node is a root node. Note here that if the parentId type is + // String, the equals() method should be used to determine. + if (-1 == UdfTreeEntity.getParent()) { + rootudfTreeList.add(UdfTreeEntity); + } + } + return rootudfTreeList; + } + + /** + * Build a tree structure according to each top-level node (root node) + * + * @return Build the entire tree + */ + public List buildTree() { + // UdfTreeEntities: Saves the complete tree structure constructed by a top-level node + List UdfTreeEntitys = new ArrayList(); + // GetRootNode(): Get all root nodes + for (UdfTreeEntity treeRootNode : getRootNode()) { + // Build subtrees from top-level nodes + treeRootNode = buildChildTree(treeRootNode); + // Complete the tree structure constructed by a top-level node and add it in + UdfTreeEntitys.add(treeRootNode); + } + return UdfTreeEntitys; + } + + /** + * Recursion ----- construct sub tree structure + * + * @param udfTreeEntity Root node (top-level node) + * @return Whole tree + */ + public UdfTreeEntity buildChildTree(UdfTreeEntity udfTreeEntity) { + List childTree = new ArrayList(); + // udfTreeList：All node sets (all data) + for (UdfTreeEntity UdfTreeEntity : udfTreeList) { + // Determine whether the parent node ID of the current node is equal to the ID of the root + // node, that is, if the current node is a child node under it + if (UdfTreeEntity.getParent().equals(udfTreeEntity.getId())) { + // Recursively judge the current node's situation and call its own method + childTree.add(buildChildTree(UdfTreeEntity)); + } + } + // Recursively judge the current node's situation and call its own method + udfTreeEntity.setChildrenList(childTree); + return udfTreeEntity; + } +} diff --git a/linkis-public-enhancements/linkis-basedata-manager/src/test/resources/create.sql b/linkis-public-enhancements/linkis-basedata-manager/src/test/resources/create.sql index 669be86918..74783c2dc2 100644 --- a/linkis-public-enhancements/linkis-basedata-manager/src/test/resources/create.sql +++ b/linkis-public-enhancements/linkis-basedata-manager/src/test/resources/create.sql @@ -128,12 +128,13 @@ CREATE TABLE `linkis_cg_rm_external_resource_provider` DROP TABLE IF EXISTS `linkis_ps_udf_manager`; -CREATE TABLE `linkis_ps_udf_manager` -( - `id` bigint(20) NOT NULL AUTO_INCREMENT, - `user_name` varchar(20) DEFAULT NULL, - PRIMARY KEY (`id`) -); +CREATE TABLE `linkis_ps_udf_manager` ( + `id` bigint(20) NOT NULL AUTO_INCREMENT, + `user_name` varchar(20) DEFAULT NULL, + `update_time` datetime DEFAULT CURRENT_TIMESTAMP, + `create_time` datetime DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY (`id`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8; DROP TABLE IF EXISTS `linkis_ps_udf_tree`; CREATE TABLE `linkis_ps_udf_tree` From 4428f71ff5dee0d9edaa1dfbf27fdb79e3c9ccd1 Mon Sep 17 00:00:00 2001 From: ahaoyao <129247228+ahaoyao@users.noreply.github.com> Date: Thu, 11 May 2023 17:18:44 +0800 Subject: [PATCH 106/261] Fix duplicate error codes --- docs/errorcode/python-errorcode.md | 10 +++++----- .../python/errorcode/LinkisPythonErrorCodeSummary.java | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/docs/errorcode/python-errorcode.md b/docs/errorcode/python-errorcode.md index 8a4f843484..cf63aaa3fe 100644 --- a/docs/errorcode/python-errorcode.md +++ b/docs/errorcode/python-errorcode.md @@ -1,7 +1,7 @@ ## python errorcode -| 模块名(服务名) | 错误码 | 描述 |enumeration name| Exception Class| -| -------- | -------- | ----- |-----|-----| -|python|41001| |PYTHON_EXECUTE_ERROR|LinkisPythonErrorCodeSummary| -|python|60003|Pyspark process has stopped, query failed!(Pyspark 进程已停止，查询失败！)|PYSPARK_PROCESSS_STOPPED|LinkisPythonErrorCodeSummary| -|python|400201|Invalid python session.(无效的 python 会话.)|INVALID_PYTHON_SESSION|LinkisPythonErrorCodeSummary| +| 模块名(服务名) | 错误码 | 描述 |enumeration name| Exception Class| +| -------- |--------| ----- |-----|-----| +|python| 60002 | |PYTHON_EXECUTE_ERROR|LinkisPythonErrorCodeSummary| +|python| 60003 |Pyspark process has stopped, query failed!(Pyspark 进程已停止，查询失败！)|PYSPARK_PROCESSS_STOPPED|LinkisPythonErrorCodeSummary| +|python| 400201 |Invalid python session.(无效的 python 会话.)|INVALID_PYTHON_SESSION|LinkisPythonErrorCodeSummary| diff --git a/linkis-engineconn-plugins/python/src/main/java/org/apache/linkis/manager/engineplugin/python/errorcode/LinkisPythonErrorCodeSummary.java b/linkis-engineconn-plugins/python/src/main/java/org/apache/linkis/manager/engineplugin/python/errorcode/LinkisPythonErrorCodeSummary.java index c17fc81e9d..9158ae6a8a 100644 --- a/linkis-engineconn-plugins/python/src/main/java/org/apache/linkis/manager/engineplugin/python/errorcode/LinkisPythonErrorCodeSummary.java +++ b/linkis-engineconn-plugins/python/src/main/java/org/apache/linkis/manager/engineplugin/python/errorcode/LinkisPythonErrorCodeSummary.java @@ -20,7 +20,7 @@ import org.apache.linkis.common.errorcode.LinkisErrorCode; public enum LinkisPythonErrorCodeSummary implements LinkisErrorCode { - PYTHON_EXECUTE_ERROR(41001, ""), + PYTHON_EXECUTE_ERROR(60002, ""), PYSPARK_PROCESSS_STOPPED( 60003, "Pyspark process has stopped, query failed!(Pyspark 进程已停止，查询失败！)"), INVALID_PYTHON_SESSION(400201, "Invalid python session.(无效的 python 会话.)"); From e76abd16126df862fb00ac4be7797019b6f42103 Mon Sep 17 00:00:00 2001 From: casionone Date: Fri, 12 May 2023 19:18:28 +0800 Subject: [PATCH 107/261] remove duplicate dependency of disruptor --- linkis-engineconn-plugins/sqoop/pom.xml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/linkis-engineconn-plugins/sqoop/pom.xml b/linkis-engineconn-plugins/sqoop/pom.xml index b5e839a8bd..1753d21654 100644 --- a/linkis-engineconn-plugins/sqoop/pom.xml +++ b/linkis-engineconn-plugins/sqoop/pom.xml @@ -66,12 +66,6 @@ 1.10.2 provided - - com.lmax - disruptor - 3.4.3 - provided - org.apache.hive hive-common From 59ff4cc87d8aabb3d6f86d39acfc803a67b9e247 Mon Sep 17 00:00:00 2001 From: peacewong Date: Mon, 15 May 2023 20:00:00 +0800 Subject: [PATCH 108/261] JDBC Optimize --- ...ErrorCode.java => LinkisSQLErrorCode.java} | 11 +- .../ujes/jdbc/LinkisSQLConnection.scala | 143 +++++--- ...ception.scala => LinkisSQLException.scala} | 27 +- .../jdbc/LinkisSQLPreparedStatement.scala | 70 ++-- .../linkis/ujes/jdbc/LinkisSQLStatement.scala | 107 +++--- .../ujes/jdbc/UJESSQLDatabaseMetaData.scala | 183 +++++----- .../linkis/ujes/jdbc/UJESSQLDriverMain.scala | 9 +- .../linkis/ujes/jdbc/UJESSQLResultSet.scala | 342 +++++++++--------- .../ujes/jdbc/UJESSQLResultSetMetaData.scala | 26 +- .../linkis/ujes/jdbc/UJESSQLTypeParser.scala | 6 +- .../jdbc/UJESSQLDatabaseMetaDataTest.java | 29 +- 11 files changed, 526 insertions(+), 427 deletions(-) rename linkis-computation-governance/linkis-jdbc-driver/src/main/java/org/apache/linkis/ujes/jdbc/{UJESSQLErrorCode.java => LinkisSQLErrorCode.java} (90%) rename linkis-computation-governance/linkis-jdbc-driver/src/main/scala/org/apache/linkis/ujes/jdbc/{UJESSQLException.scala => LinkisSQLException.scala} (61%) diff --git a/linkis-computation-governance/linkis-jdbc-driver/src/main/java/org/apache/linkis/ujes/jdbc/UJESSQLErrorCode.java b/linkis-computation-governance/linkis-jdbc-driver/src/main/java/org/apache/linkis/ujes/jdbc/LinkisSQLErrorCode.java similarity index 90% rename from linkis-computation-governance/linkis-jdbc-driver/src/main/java/org/apache/linkis/ujes/jdbc/UJESSQLErrorCode.java rename to linkis-computation-governance/linkis-jdbc-driver/src/main/java/org/apache/linkis/ujes/jdbc/LinkisSQLErrorCode.java index fc283d8fbe..442cbbfb43 100644 --- a/linkis-computation-governance/linkis-jdbc-driver/src/main/java/org/apache/linkis/ujes/jdbc/UJESSQLErrorCode.java +++ b/linkis-computation-governance/linkis-jdbc-driver/src/main/java/org/apache/linkis/ujes/jdbc/LinkisSQLErrorCode.java @@ -17,7 +17,7 @@ package org.apache.linkis.ujes.jdbc; -public enum UJESSQLErrorCode { +public enum LinkisSQLErrorCode { BAD_URL(80000, "bad url"), NOSUPPORT_DRIVER(80001, "this method not supported in driver"), NOSUPPORT_CONNECTION(80002, "this method not supported in connection"), @@ -38,11 +38,12 @@ public enum UJESSQLErrorCode { RESULTSET_NULL( 80017, "resultset is null,try to run next() firstly to init ResultSet and MetaData"), PREPARESTATEMENT_TYPEERROR(80018, "parameter type error"), - METADATA_EMPTY(80019, "data is empty"); + METADATA_EMPTY(80019, "data is empty"), + UNKNOWN_ERROR(80020, "unknown error"); private String msg; private int code; - UJESSQLErrorCode(int code, String msg) { + LinkisSQLErrorCode(int code, String msg) { this.code = code; this.msg = msg; } @@ -54,4 +55,8 @@ public String getMsg() { public int getCode() { return code; } + + public void setMsg(String msg) { + this.msg = msg; + } } diff --git a/linkis-computation-governance/linkis-jdbc-driver/src/main/scala/org/apache/linkis/ujes/jdbc/LinkisSQLConnection.scala b/linkis-computation-governance/linkis-jdbc-driver/src/main/scala/org/apache/linkis/ujes/jdbc/LinkisSQLConnection.scala index 6683161b8e..b800698766 100644 --- a/linkis-computation-governance/linkis-jdbc-driver/src/main/scala/org/apache/linkis/ujes/jdbc/LinkisSQLConnection.scala +++ b/linkis-computation-governance/linkis-jdbc-driver/src/main/scala/org/apache/linkis/ujes/jdbc/LinkisSQLConnection.scala @@ -122,7 +122,7 @@ class LinkisSQLConnection(private[jdbc] val ujesClient: UJESClient, props: Prope } private[jdbc] def throwWhenClosed[T](op: => T): T = - if (isClosed) throw new UJESSQLException(UJESSQLErrorCode.CONNECTION_CLOSED) + if (isClosed) throw new LinkisSQLException(LinkisSQLErrorCode.CONNECTION_CLOSED) else op private def createStatementAndAdd[T <: Statement](op: => T): T = throwWhenClosed { @@ -183,7 +183,10 @@ class LinkisSQLConnection(private[jdbc] val ujesClient: UJESClient, props: Prope override def isClosed: Boolean = closed override def setReadOnly(readOnly: Boolean): Unit = - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_CONNECTION, "setReadOnly not supported") + throw new LinkisSQLException( + LinkisSQLErrorCode.NOSUPPORT_CONNECTION, + "setReadOnly not supported" + ) override def isReadOnly: Boolean = false @@ -206,45 +209,63 @@ class LinkisSQLConnection(private[jdbc] val ujesClient: UJESClient, props: Prope override def commit(): Unit = {} override def prepareCall(sql: String): CallableStatement = - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_CONNECTION, "prepareCall not supported") + throw new LinkisSQLException( + LinkisSQLErrorCode.NOSUPPORT_CONNECTION, + "prepareCall not supported" + ) override def rollback(): Unit = - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_CONNECTION, "rollback not supported") + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_CONNECTION, "rollback not supported") override def nativeSQL(sql: String): String = - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_CONNECTION, "nativeSQL not supported") + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_CONNECTION, "nativeSQL not supported") override def prepareCall( sql: String, resultSetType: Int, resultSetConcurrency: Int ): CallableStatement = - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_CONNECTION, "prepareCall not supported") + throw new LinkisSQLException( + LinkisSQLErrorCode.NOSUPPORT_CONNECTION, + "prepareCall not supported" + ) override def getTypeMap: util.Map[String, Class[_]] = - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_CONNECTION, "getTypeMap not supported") + throw new LinkisSQLException( + LinkisSQLErrorCode.NOSUPPORT_CONNECTION, + "getTypeMap not supported" + ) override def setTypeMap(map: util.Map[String, Class[_]]): Unit = - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_CONNECTION, "setTypeMap not supported") + throw new LinkisSQLException( + LinkisSQLErrorCode.NOSUPPORT_CONNECTION, + "setTypeMap not supported" + ) - override def setHoldability(holdability: Int): Unit = throw new UJESSQLException( - UJESSQLErrorCode.NOSUPPORT_CONNECTION, + override def setHoldability(holdability: Int): Unit = throw new LinkisSQLException( + LinkisSQLErrorCode.NOSUPPORT_CONNECTION, "setHoldability not supported" ) override def getHoldability: Int = 0 override def setSavepoint(): Savepoint = - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_CONNECTION, "setSavepoint not supported") + throw new LinkisSQLException( + LinkisSQLErrorCode.NOSUPPORT_CONNECTION, + "setSavepoint not supported" + ) override def setSavepoint(name: String): Savepoint = - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_CONNECTION, "setSavepoint not supported") + throw new LinkisSQLException( + LinkisSQLErrorCode.NOSUPPORT_CONNECTION, + "setSavepoint not supported" + ) override def rollback(savepoint: Savepoint): Unit = - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_CONNECTION, "rollback not supported") + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_CONNECTION, "rollback not supported") - override def releaseSavepoint(savepoint: Savepoint): Unit = throw new UJESSQLException( - UJESSQLErrorCode.NOSUPPORT_CONNECTION, + override def releaseSavepoint(savepoint: Savepoint): Unit = throw new LinkisSQLException( + LinkisSQLErrorCode.NOSUPPORT_CONNECTION, "releaseSavepoint not supported" ) @@ -253,8 +274,8 @@ class LinkisSQLConnection(private[jdbc] val ujesClient: UJESClient, props: Prope resultSetConcurrency: Int, resultSetHoldability: Int ): Statement = - throw new UJESSQLException( - UJESSQLErrorCode.NOSUPPORT_CONNECTION, + throw new LinkisSQLException( + LinkisSQLErrorCode.NOSUPPORT_CONNECTION, "createStatement not supported" ) @@ -264,8 +285,8 @@ class LinkisSQLConnection(private[jdbc] val ujesClient: UJESClient, props: Prope resultSetConcurrency: Int, resultSetHoldability: Int ): PreparedStatement = - throw new UJESSQLException( - UJESSQLErrorCode.NOSUPPORT_CONNECTION, + throw new LinkisSQLException( + LinkisSQLErrorCode.NOSUPPORT_CONNECTION, "prepareStatement not supported" ) @@ -275,55 +296,88 @@ class LinkisSQLConnection(private[jdbc] val ujesClient: UJESClient, props: Prope resultSetConcurrency: Int, resultSetHoldability: Int ): CallableStatement = - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_CONNECTION, "prepareCall not supported") + throw new LinkisSQLException( + LinkisSQLErrorCode.NOSUPPORT_CONNECTION, + "prepareCall not supported" + ) override def prepareStatement(sql: String, columnIndexes: Array[Int]): PreparedStatement = - throw new UJESSQLException( - UJESSQLErrorCode.NOSUPPORT_CONNECTION, + throw new LinkisSQLException( + LinkisSQLErrorCode.NOSUPPORT_CONNECTION, "prepareStatement not supported" ) override def prepareStatement(sql: String, columnNames: Array[String]): PreparedStatement = - throw new UJESSQLException( - UJESSQLErrorCode.NOSUPPORT_CONNECTION, + throw new LinkisSQLException( + LinkisSQLErrorCode.NOSUPPORT_CONNECTION, "prepareStatement not supported" ) override def createClob(): Clob = - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_CONNECTION, "createClob not supported") + throw new LinkisSQLException( + LinkisSQLErrorCode.NOSUPPORT_CONNECTION, + "createClob not supported" + ) override def createBlob(): Blob = - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_CONNECTION, "createBlob not supported") + throw new LinkisSQLException( + LinkisSQLErrorCode.NOSUPPORT_CONNECTION, + "createBlob not supported" + ) override def createNClob(): NClob = - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_CONNECTION, "createNClob not supported") + throw new LinkisSQLException( + LinkisSQLErrorCode.NOSUPPORT_CONNECTION, + "createNClob not supported" + ) override def createSQLXML(): SQLXML = - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_CONNECTION, "createSQLXML not supported") + throw new LinkisSQLException( + LinkisSQLErrorCode.NOSUPPORT_CONNECTION, + "createSQLXML not supported" + ) override def isValid(timeout: Int): Boolean = true override def setClientInfo(name: String, value: String): Unit = - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_CONNECTION, "setClientInfo not supported") + throw new LinkisSQLException( + LinkisSQLErrorCode.NOSUPPORT_CONNECTION, + "setClientInfo not supported" + ) override def setClientInfo(properties: Properties): Unit = - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_CONNECTION, "properties not supported") + throw new LinkisSQLException( + LinkisSQLErrorCode.NOSUPPORT_CONNECTION, + "properties not supported" + ) override def getClientInfo(name: String): String = - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_CONNECTION, "getClientInfo not supported") + throw new LinkisSQLException( + LinkisSQLErrorCode.NOSUPPORT_CONNECTION, + "getClientInfo not supported" + ) override def getClientInfo: Properties = - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_CONNECTION, "getClientInfo not supported") + throw new LinkisSQLException( + LinkisSQLErrorCode.NOSUPPORT_CONNECTION, + "getClientInfo not supported" + ) override def createArrayOf(typeName: String, elements: Array[AnyRef]): sql.Array = - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_CONNECTION, "createArrayOf not supported") + throw new LinkisSQLException( + LinkisSQLErrorCode.NOSUPPORT_CONNECTION, + "createArrayOf not supported" + ) override def createStruct(typeName: String, attributes: Array[AnyRef]): Struct = - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_CONNECTION, "createStruct not supported") + throw new LinkisSQLException( + LinkisSQLErrorCode.NOSUPPORT_CONNECTION, + "createStruct not supported" + ) override def setSchema(schema: String): Unit = throwWhenClosed { if (StringUtils.isBlank(schema)) { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_STATEMENT, "schema is empty!") + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_STATEMENT, "schema is empty!") } createStatement().execute("use " + schema) } @@ -331,30 +385,33 @@ class LinkisSQLConnection(private[jdbc] val ujesClient: UJESClient, props: Prope override def getSchema: String = throwWhenClosed { val resultSet = createStatement().executeQuery("SELECT current_database()") if (!resultSet.next()) { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_STATEMENT, "Get schema failed!") + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_STATEMENT, "Get schema failed!") } resultSet.getString(1) } override def abort(executor: Executor): Unit = - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_CONNECTION, "abort not supported") + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_CONNECTION, "abort not supported") override def setNetworkTimeout(executor: Executor, milliseconds: Int): Unit = - throw new UJESSQLException( - UJESSQLErrorCode.NOSUPPORT_CONNECTION, + throw new LinkisSQLException( + LinkisSQLErrorCode.NOSUPPORT_CONNECTION, "setNetworkTimeout not supported" ) - override def getNetworkTimeout: Int = throw new UJESSQLException( - UJESSQLErrorCode.NOSUPPORT_CONNECTION, + override def getNetworkTimeout: Int = throw new LinkisSQLException( + LinkisSQLErrorCode.NOSUPPORT_CONNECTION, "getNetworkTimeout not supported" ) override def unwrap[T](iface: Class[T]): T = - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_CONNECTION, "unwrap not supported") + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_CONNECTION, "unwrap not supported") override def isWrapperFor(iface: Class[_]): Boolean = - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_CONNECTION, "isWrapperFor not supported") + throw new LinkisSQLException( + LinkisSQLErrorCode.NOSUPPORT_CONNECTION, + "isWrapperFor not supported" + ) def addLabels(labels: util.Map[String, AnyRef]): Unit = { labelMap.putAll(labels) diff --git a/linkis-computation-governance/linkis-jdbc-driver/src/main/scala/org/apache/linkis/ujes/jdbc/UJESSQLException.scala b/linkis-computation-governance/linkis-jdbc-driver/src/main/scala/org/apache/linkis/ujes/jdbc/LinkisSQLException.scala similarity index 61% rename from linkis-computation-governance/linkis-jdbc-driver/src/main/scala/org/apache/linkis/ujes/jdbc/UJESSQLException.scala rename to linkis-computation-governance/linkis-jdbc-driver/src/main/scala/org/apache/linkis/ujes/jdbc/LinkisSQLException.scala index 25db6f9381..9cc3f3814f 100644 --- a/linkis-computation-governance/linkis-jdbc-driver/src/main/scala/org/apache/linkis/ujes/jdbc/UJESSQLException.scala +++ b/linkis-computation-governance/linkis-jdbc-driver/src/main/scala/org/apache/linkis/ujes/jdbc/LinkisSQLException.scala @@ -17,26 +17,21 @@ package org.apache.linkis.ujes.jdbc -import org.apache.linkis.common.exception.ErrorException +import java.sql.SQLException -class UJESSQLException(errorCode: UJESSQLErrorCode) - extends ErrorException(errorCode.getCode, errorCode.getMsg) { +class LinkisSQLException(msg: String, code: String, vendorCode: Int) + extends SQLException(msg, code, vendorCode) { - def this(errorCode: UJESSQLErrorCode, msg: String) { - this(errorCode) - setErrCode(errorCode.getCode) - setDesc(msg) + def this(errorCode: LinkisSQLErrorCode, msg: String) { + this(msg, errorCode.getCode.toString, 0) } - /** - * add to deal with errorinfo derived from jobInfo - * @param errorCode - * @param msg - */ - def this(errorCode: Int, msg: String) { - this(UJESSQLErrorCode.ERRORINFO_FROM_JOBINFO) - setDesc(msg) - setErrCode(errorCode) + def this(errorCode: LinkisSQLErrorCode) { + this(errorCode.getMsg, errorCode.getCode.toString, 0) + } + + def this(msg: String, code: String) { + this(msg, code, 0) } } diff --git a/linkis-computation-governance/linkis-jdbc-driver/src/main/scala/org/apache/linkis/ujes/jdbc/LinkisSQLPreparedStatement.scala b/linkis-computation-governance/linkis-jdbc-driver/src/main/scala/org/apache/linkis/ujes/jdbc/LinkisSQLPreparedStatement.scala index 326a16bc7c..61a7020946 100644 --- a/linkis-computation-governance/linkis-jdbc-driver/src/main/scala/org/apache/linkis/ujes/jdbc/LinkisSQLPreparedStatement.scala +++ b/linkis-computation-governance/linkis-jdbc-driver/src/main/scala/org/apache/linkis/ujes/jdbc/LinkisSQLPreparedStatement.scala @@ -139,7 +139,7 @@ class LinkisSQLPreparedStatement(ujesSQLConnection: LinkisSQLConnection, sql: St } override def setTime(parameterIndex: Int, x: Time): Unit = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_STATEMENT) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_STATEMENT) } override def setTimestamp(parameterIndex: Int, x: Timestamp): Unit = { @@ -147,15 +147,15 @@ class LinkisSQLPreparedStatement(ujesSQLConnection: LinkisSQLConnection, sql: St } override def setAsciiStream(parameterIndex: Int, x: InputStream, length: Int): Unit = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_STATEMENT) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_STATEMENT) } override def setUnicodeStream(parameterIndex: Int, x: InputStream, length: Int): Unit = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_STATEMENT) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_STATEMENT) } override def setBinaryStream(parameterIndex: Int, x: InputStream, length: Int): Unit = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_STATEMENT) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_STATEMENT) } override def clearParameters(): Unit = { @@ -163,7 +163,7 @@ class LinkisSQLPreparedStatement(ujesSQLConnection: LinkisSQLConnection, sql: St } override def setObject(parameterIndex: Int, x: scala.Any, targetSqlType: Int): Unit = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_STATEMENT) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_STATEMENT) } override def setObject(parameterIndex: Int, x: scala.Any): Unit = { @@ -180,8 +180,8 @@ class LinkisSQLPreparedStatement(ujesSQLConnection: LinkisSQLConnection, sql: St case x: Char => setString(parameterIndex, x.toString) case x: Timestamp => setTimestamp(parameterIndex, x) case _ => - throw new UJESSQLException( - UJESSQLErrorCode.PREPARESTATEMENT_TYPEERROR, + throw new LinkisSQLException( + LinkisSQLErrorCode.PREPARESTATEMENT_TYPEERROR, s"Can''t infer the SQL type to use for an instance of ${x.getClass.getName}. Use setObject() with an explicit Types value to specify the type to use" ) } @@ -208,23 +208,23 @@ class LinkisSQLPreparedStatement(ujesSQLConnection: LinkisSQLConnection, sql: St } override def setCharacterStream(parameterIndex: Int, reader: Reader, length: Int): Unit = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_STATEMENT) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_STATEMENT) } override def setRef(parameterIndex: Int, x: Ref): Unit = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_STATEMENT) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_STATEMENT) } override def setBlob(parameterIndex: Int, x: Blob): Unit = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_STATEMENT) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_STATEMENT) } override def setClob(parameterIndex: Int, x: Clob): Unit = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_STATEMENT) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_STATEMENT) } override def setArray(parameterIndex: Int, x: java.sql.Array): Unit = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_STATEMENT) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_STATEMENT) } override def getMetaData: ResultSetMetaData = { @@ -235,15 +235,15 @@ class LinkisSQLPreparedStatement(ujesSQLConnection: LinkisSQLConnection, sql: St } override def setDate(parameterIndex: Int, x: Date, cal: Calendar): Unit = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_STATEMENT) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_STATEMENT) } override def setTime(parameterIndex: Int, x: Time, cal: Calendar): Unit = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_STATEMENT) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_STATEMENT) } override def setTimestamp(parameterIndex: Int, x: Timestamp, cal: Calendar): Unit = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_STATEMENT) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_STATEMENT) } override def setNull(parameterIndex: Int, sqlType: Int, typeName: String): Unit = { @@ -251,7 +251,7 @@ class LinkisSQLPreparedStatement(ujesSQLConnection: LinkisSQLConnection, sql: St } override def setURL(parameterIndex: Int, x: URL): Unit = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_STATEMENT) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_STATEMENT) } override def getParameterMetaData: ParameterMetaData = { @@ -264,35 +264,35 @@ class LinkisSQLPreparedStatement(ujesSQLConnection: LinkisSQLConnection, sql: St } override def setRowId(parameterIndex: Int, x: RowId): Unit = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_STATEMENT) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_STATEMENT) } override def setNString(parameterIndex: Int, value: String): Unit = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_STATEMENT) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_STATEMENT) } override def setNCharacterStream(parameterIndex: Int, value: Reader, length: Long): Unit = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_STATEMENT) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_STATEMENT) } override def setNClob(parameterIndex: Int, value: NClob): Unit = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_STATEMENT) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_STATEMENT) } override def setClob(parameterIndex: Int, reader: Reader, length: Long): Unit = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_STATEMENT) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_STATEMENT) } override def setBlob(parameterIndex: Int, inputStream: InputStream, length: Long): Unit = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_STATEMENT) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_STATEMENT) } override def setNClob(parameterIndex: Int, reader: Reader, length: Long): Unit = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_STATEMENT) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_STATEMENT) } override def setSQLXML(parameterIndex: Int, xmlObject: SQLXML): Unit = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_STATEMENT) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_STATEMENT) } override def setObject( @@ -301,47 +301,47 @@ class LinkisSQLPreparedStatement(ujesSQLConnection: LinkisSQLConnection, sql: St targetSqlType: Int, scaleOrLength: Int ): Unit = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_STATEMENT) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_STATEMENT) } override def setAsciiStream(parameterIndex: Int, x: InputStream, length: Long): Unit = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_STATEMENT) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_STATEMENT) } override def setBinaryStream(parameterIndex: Int, x: InputStream, length: Long): Unit = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_STATEMENT) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_STATEMENT) } override def setCharacterStream(parameterIndex: Int, reader: Reader, length: Long): Unit = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_STATEMENT) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_STATEMENT) } override def setAsciiStream(parameterIndex: Int, x: InputStream): Unit = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_STATEMENT) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_STATEMENT) } override def setBinaryStream(parameterIndex: Int, x: InputStream): Unit = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_STATEMENT) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_STATEMENT) } override def setCharacterStream(parameterIndex: Int, reader: Reader): Unit = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_STATEMENT) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_STATEMENT) } override def setNCharacterStream(parameterIndex: Int, value: Reader): Unit = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_STATEMENT) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_STATEMENT) } override def setClob(parameterIndex: Int, reader: Reader): Unit = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_STATEMENT) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_STATEMENT) } override def setBlob(parameterIndex: Int, inputStream: InputStream): Unit = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_STATEMENT) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_STATEMENT) } override def setNClob(parameterIndex: Int, reader: Reader): Unit = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_STATEMENT) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_STATEMENT) } override def getResultSetType: Int = { diff --git a/linkis-computation-governance/linkis-jdbc-driver/src/main/scala/org/apache/linkis/ujes/jdbc/LinkisSQLStatement.scala b/linkis-computation-governance/linkis-jdbc-driver/src/main/scala/org/apache/linkis/ujes/jdbc/LinkisSQLStatement.scala index 1f6d5fc114..88cf3013ca 100644 --- a/linkis-computation-governance/linkis-jdbc-driver/src/main/scala/org/apache/linkis/ujes/jdbc/LinkisSQLStatement.scala +++ b/linkis-computation-governance/linkis-jdbc-driver/src/main/scala/org/apache/linkis/ujes/jdbc/LinkisSQLStatement.scala @@ -17,7 +17,6 @@ package org.apache.linkis.ujes.jdbc -import org.apache.linkis.common.exception.ErrorException import org.apache.linkis.common.utils.{Logging, Utils} import org.apache.linkis.governance.common.entity.ExecutionNodeStatus import org.apache.linkis.ujes.client.request.OpenLogAction @@ -26,7 +25,7 @@ import org.apache.linkis.ujes.jdbc.hook.JDBCDriverPreExecutionHook import org.apache.commons.lang3.StringUtils -import java.sql.{Connection, ResultSet, SQLWarning, Statement} +import java.sql._ import java.util import java.util.concurrent.TimeUnit @@ -52,12 +51,12 @@ class LinkisSQLStatement(private[jdbc] val ujesSQLConnection: LinkisSQLConnectio private val logSize = 100 private[jdbc] def throwWhenClosed[T](op: => T): T = ujesSQLConnection.throwWhenClosed { - if (isClosed) throw new UJESSQLException(UJESSQLErrorCode.STATEMENT_CLOSED) + if (isClosed) throw new LinkisSQLException(LinkisSQLErrorCode.STATEMENT_CLOSED) else op } override def executeQuery(sql: String): UJESSQLResultSet = { - if (!execute(sql)) throw new UJESSQLException(UJESSQLErrorCode.RESULTSET_NULL) + if (!execute(sql)) throw new LinkisSQLException(LinkisSQLErrorCode.RESULTSET_NULL) resultSet } @@ -82,13 +81,13 @@ class LinkisSQLStatement(private[jdbc] val ujesSQLConnection: LinkisSQLConnectio } } - override def getMaxFieldSize: Int = throw new UJESSQLException( - UJESSQLErrorCode.NOSUPPORT_STATEMENT, + override def getMaxFieldSize: Int = throw new LinkisSQLException( + LinkisSQLErrorCode.NOSUPPORT_STATEMENT, "getMaxFieldSize not supported" ) - override def setMaxFieldSize(max: Int): Unit = throw new UJESSQLException( - UJESSQLErrorCode.NOSUPPORT_STATEMENT, + override def setMaxFieldSize(max: Int): Unit = throw new LinkisSQLException( + LinkisSQLErrorCode.NOSUPPORT_STATEMENT, "setMaxFieldSize not supported" ) @@ -97,8 +96,8 @@ class LinkisSQLStatement(private[jdbc] val ujesSQLConnection: LinkisSQLConnectio override def setMaxRows(max: Int): Unit = this.maxRows = max override def setEscapeProcessing(enable: Boolean): Unit = if (enable) { - throw new UJESSQLException( - UJESSQLErrorCode.NOSUPPORT_STATEMENT, + throw new LinkisSQLException( + LinkisSQLErrorCode.NOSUPPORT_STATEMENT, "setEscapeProcessing not supported" ) } @@ -114,7 +113,10 @@ class LinkisSQLStatement(private[jdbc] val ujesSQLConnection: LinkisSQLConnectio override def clearWarnings(): Unit = {} override def setCursorName(name: String): Unit = - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_STATEMENT, "setCursorName not supported") + throw new LinkisSQLException( + LinkisSQLErrorCode.NOSUPPORT_STATEMENT, + "setCursorName not supported" + ) override def execute(sql: String): Boolean = throwWhenClosed { var parsedSQL = sql @@ -124,7 +126,7 @@ class LinkisSQLStatement(private[jdbc] val ujesSQLConnection: LinkisSQLConnectio logger.info(s"begin to execute sql ${parsedSQL}") queryEnd = false logPath = null - Utils.tryFinally { + try { jobExecuteResult = ujesSQLConnection.toSubmit(parsedSQL) val atMost = if (queryTimeout > 0) Duration(queryTimeout, TimeUnit.MILLISECONDS) else Duration.Inf @@ -147,16 +149,16 @@ class LinkisSQLStatement(private[jdbc] val ujesSQLConnection: LinkisSQLConnectio case t: TimeoutException => if (queryTimeout > 0) clearQuery() logPath = jobInfo.getRequestPersistTask.getLogPath - new UJESSQLException(UJESSQLErrorCode.QUERY_TIMEOUT, "query has been timeout!") + new LinkisSQLException(LinkisSQLErrorCode.QUERY_TIMEOUT, "query has been timeout!") .initCause(t) case t => t } } logPath = jobInfo.getRequestPersistTask.getLogPath if (!ExecutionNodeStatus.isSucceed(ExecutionNodeStatus.valueOf(jobInfo.getJobStatus))) { - throw new ErrorException( - jobInfo.getRequestPersistTask.getErrCode, - jobInfo.getRequestPersistTask.getErrDesc + throw new LinkisSQLException( + jobInfo.getRequestPersistTask.getErrDesc, + jobInfo.getRequestPersistTask.getErrCode.toString ) } @@ -169,7 +171,15 @@ class LinkisSQLStatement(private[jdbc] val ujesSQLConnection: LinkisSQLConnectio } else { false } - } { + } catch { + case sqlException: SQLException => + throw sqlException + case throwable: Throwable => + val exception = + new LinkisSQLException(LinkisSQLErrorCode.UNKNOWN_ERROR, throwable.getMessage) + exception.initCause(throwable) + throw exception + } finally { queryEnd = true } } @@ -184,8 +194,8 @@ class LinkisSQLStatement(private[jdbc] val ujesSQLConnection: LinkisSQLConnectio override def setFetchDirection(direction: Int): Unit = throwWhenClosed(if (direction != ResultSet.FETCH_FORWARD) { - throw new UJESSQLException( - UJESSQLErrorCode.NOSUPPORT_STATEMENT, + throw new LinkisSQLException( + LinkisSQLErrorCode.NOSUPPORT_STATEMENT, "only FETCH_FORWARD is supported." ) }) @@ -196,88 +206,97 @@ class LinkisSQLStatement(private[jdbc] val ujesSQLConnection: LinkisSQLConnectio override def getFetchSize: Int = fetchSize - override def getResultSetConcurrency: Int = throw new UJESSQLException( - UJESSQLErrorCode.NOSUPPORT_STATEMENT, + override def getResultSetConcurrency: Int = throw new LinkisSQLException( + LinkisSQLErrorCode.NOSUPPORT_STATEMENT, "getResultSetConcurrency not supported." ) override def getResultSetType: Int = throwWhenClosed(ResultSet.TYPE_FORWARD_ONLY) override def addBatch(sql: String): Unit = - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_STATEMENT, "addBatch not supported.") + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_STATEMENT, "addBatch not supported.") override def clearBatch(): Unit = - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_STATEMENT, "clearBatch not supported.") + throw new LinkisSQLException( + LinkisSQLErrorCode.NOSUPPORT_STATEMENT, + "clearBatch not supported." + ) override def executeBatch(): Array[Int] = - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_STATEMENT, "executeBatch not supported.") + throw new LinkisSQLException( + LinkisSQLErrorCode.NOSUPPORT_STATEMENT, + "executeBatch not supported." + ) override def getConnection: Connection = throwWhenClosed(ujesSQLConnection) override def getMoreResults(current: Int): Boolean = false - override def getGeneratedKeys: ResultSet = throw new UJESSQLException( - UJESSQLErrorCode.NOSUPPORT_STATEMENT, + override def getGeneratedKeys: ResultSet = throw new LinkisSQLException( + LinkisSQLErrorCode.NOSUPPORT_STATEMENT, "getGeneratedKeys not supported." ) override def executeUpdate(sql: String, autoGeneratedKeys: Int): Int = - throw new UJESSQLException( - UJESSQLErrorCode.NOSUPPORT_STATEMENT, + throw new LinkisSQLException( + LinkisSQLErrorCode.NOSUPPORT_STATEMENT, "executeUpdate with autoGeneratedKeys not supported." ) override def executeUpdate(sql: String, columnIndexes: Array[Int]): Int = - throw new UJESSQLException( - UJESSQLErrorCode.NOSUPPORT_STATEMENT, + throw new LinkisSQLException( + LinkisSQLErrorCode.NOSUPPORT_STATEMENT, "executeUpdate with columnIndexes not supported." ) override def executeUpdate(sql: String, columnNames: Array[String]): Int = - throw new UJESSQLException( - UJESSQLErrorCode.NOSUPPORT_STATEMENT, + throw new LinkisSQLException( + LinkisSQLErrorCode.NOSUPPORT_STATEMENT, "executeUpdate with columnNames not supported." ) override def execute(sql: String, autoGeneratedKeys: Int): Boolean = - throw new UJESSQLException( - UJESSQLErrorCode.NOSUPPORT_STATEMENT, + throw new LinkisSQLException( + LinkisSQLErrorCode.NOSUPPORT_STATEMENT, "execute with autoGeneratedKeys not supported." ) override def execute(sql: String, columnIndexes: Array[Int]): Boolean = - throw new UJESSQLException( - UJESSQLErrorCode.NOSUPPORT_STATEMENT, + throw new LinkisSQLException( + LinkisSQLErrorCode.NOSUPPORT_STATEMENT, "execute with columnIndexes not supported." ) override def execute(sql: String, columnNames: Array[String]): Boolean = - throw new UJESSQLException( - UJESSQLErrorCode.NOSUPPORT_STATEMENT, + throw new LinkisSQLException( + LinkisSQLErrorCode.NOSUPPORT_STATEMENT, "execute with columnNames not supported." ) - override def getResultSetHoldability: Int = throw new UJESSQLException( - UJESSQLErrorCode.NOSUPPORT_STATEMENT, + override def getResultSetHoldability: Int = throw new LinkisSQLException( + LinkisSQLErrorCode.NOSUPPORT_STATEMENT, "getResultSetHoldability not supported" ) override def isClosed: Boolean = closed override def setPoolable(poolable: Boolean): Unit = - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_STATEMENT, "setPoolable not supported") + throw new LinkisSQLException( + LinkisSQLErrorCode.NOSUPPORT_STATEMENT, + "setPoolable not supported" + ) override def isPoolable: Boolean = false - override def closeOnCompletion(): Unit = throw new UJESSQLException( - UJESSQLErrorCode.NOSUPPORT_STATEMENT, + override def closeOnCompletion(): Unit = throw new LinkisSQLException( + LinkisSQLErrorCode.NOSUPPORT_STATEMENT, "closeOnCompletion not supported" ) override def isCloseOnCompletion: Boolean = false override def unwrap[T](iface: Class[T]): T = - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_STATEMENT, "unwrap not supported") + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_STATEMENT, "unwrap not supported") override def isWrapperFor(iface: Class[_]): Boolean = false diff --git a/linkis-computation-governance/linkis-jdbc-driver/src/main/scala/org/apache/linkis/ujes/jdbc/UJESSQLDatabaseMetaData.scala b/linkis-computation-governance/linkis-jdbc-driver/src/main/scala/org/apache/linkis/ujes/jdbc/UJESSQLDatabaseMetaData.scala index ad856e81e0..25e94c5370 100644 --- a/linkis-computation-governance/linkis-jdbc-driver/src/main/scala/org/apache/linkis/ujes/jdbc/UJESSQLDatabaseMetaData.scala +++ b/linkis-computation-governance/linkis-jdbc-driver/src/main/scala/org/apache/linkis/ujes/jdbc/UJESSQLDatabaseMetaData.scala @@ -41,7 +41,8 @@ class UJESSQLDatabaseMetaData(ujesSQLConnection: LinkisSQLConnection) override def getUserName: String = if (ujesSQLConnection.getProps.containsKey("user")) ujesSQLConnection.getProps.getProperty("user") - else throw new UJESSQLException(UJESSQLErrorCode.PARAMS_NOT_FOUND, "Missing user information") + else + throw new LinkisSQLException(LinkisSQLErrorCode.PARAMS_NOT_FOUND, "Missing user information") override def isReadOnly: Boolean = false @@ -88,7 +89,10 @@ class UJESSQLDatabaseMetaData(ujesSQLConnection: LinkisSQLConnection) override def getIdentifierQuoteString: String = " " override def getSQLKeywords: String = - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_METADATA, "getSQLKeywords not supported") + throw new LinkisSQLException( + LinkisSQLErrorCode.NOSUPPORT_METADATA, + "getSQLKeywords not supported" + ) override def getNumericFunctions: String = "" @@ -162,8 +166,8 @@ class UJESSQLDatabaseMetaData(ujesSQLConnection: LinkisSQLConnection) override def getCatalogTerm: String = "instance" - override def isCatalogAtStart: Boolean = throw new UJESSQLException( - UJESSQLErrorCode.NOSUPPORT_METADATA, + override def isCatalogAtStart: Boolean = throw new LinkisSQLException( + LinkisSQLErrorCode.NOSUPPORT_METADATA, "isCatalogAtStart not supported" ) @@ -219,103 +223,106 @@ class UJESSQLDatabaseMetaData(ujesSQLConnection: LinkisSQLConnection) override def supportsOpenStatementsAcrossRollback(): Boolean = false - override def getMaxBinaryLiteralLength: Int = throw new UJESSQLException( - UJESSQLErrorCode.NOSUPPORT_METADATA, + override def getMaxBinaryLiteralLength: Int = throw new LinkisSQLException( + LinkisSQLErrorCode.NOSUPPORT_METADATA, "getMaxBinaryLiteralLength not supported" ) - override def getMaxCharLiteralLength: Int = throw new UJESSQLException( - UJESSQLErrorCode.NOSUPPORT_METADATA, + override def getMaxCharLiteralLength: Int = throw new LinkisSQLException( + LinkisSQLErrorCode.NOSUPPORT_METADATA, "getMaxCharLiteralLength not supported" ) override def getMaxColumnNameLength: Int = 128 - override def getMaxColumnsInGroupBy: Int = throw new UJESSQLException( - UJESSQLErrorCode.NOSUPPORT_METADATA, + override def getMaxColumnsInGroupBy: Int = throw new LinkisSQLException( + LinkisSQLErrorCode.NOSUPPORT_METADATA, "getMaxColumnsInGroupBy not supported" ) - override def getMaxColumnsInIndex: Int = throw new UJESSQLException( - UJESSQLErrorCode.NOSUPPORT_METADATA, + override def getMaxColumnsInIndex: Int = throw new LinkisSQLException( + LinkisSQLErrorCode.NOSUPPORT_METADATA, "getMaxColumnsInIndex not supported" ) - override def getMaxColumnsInOrderBy: Int = throw new UJESSQLException( - UJESSQLErrorCode.NOSUPPORT_METADATA, + override def getMaxColumnsInOrderBy: Int = throw new LinkisSQLException( + LinkisSQLErrorCode.NOSUPPORT_METADATA, "getMaxColumnsInOrderBy not supported" ) - override def getMaxColumnsInSelect: Int = throw new UJESSQLException( - UJESSQLErrorCode.NOSUPPORT_METADATA, + override def getMaxColumnsInSelect: Int = throw new LinkisSQLException( + LinkisSQLErrorCode.NOSUPPORT_METADATA, "getMaxColumnsInSelect not supported" ) - override def getMaxColumnsInTable: Int = throw new UJESSQLException( - UJESSQLErrorCode.NOSUPPORT_METADATA, + override def getMaxColumnsInTable: Int = throw new LinkisSQLException( + LinkisSQLErrorCode.NOSUPPORT_METADATA, "getMaxColumnsInTable not supported" ) - override def getMaxConnections: Int = throw new UJESSQLException( - UJESSQLErrorCode.NOSUPPORT_METADATA, + override def getMaxConnections: Int = throw new LinkisSQLException( + LinkisSQLErrorCode.NOSUPPORT_METADATA, "getMaxConnections not supported" ) - override def getMaxCursorNameLength: Int = throw new UJESSQLException( - UJESSQLErrorCode.NOSUPPORT_METADATA, + override def getMaxCursorNameLength: Int = throw new LinkisSQLException( + LinkisSQLErrorCode.NOSUPPORT_METADATA, "getMaxCursorNameLength not supported" ) - override def getMaxIndexLength: Int = throw new UJESSQLException( - UJESSQLErrorCode.NOSUPPORT_METADATA, + override def getMaxIndexLength: Int = throw new LinkisSQLException( + LinkisSQLErrorCode.NOSUPPORT_METADATA, "getMaxIndexLength not supported" ) - override def getMaxSchemaNameLength: Int = throw new UJESSQLException( - UJESSQLErrorCode.NOSUPPORT_METADATA, + override def getMaxSchemaNameLength: Int = throw new LinkisSQLException( + LinkisSQLErrorCode.NOSUPPORT_METADATA, "getMaxSchemaNameLength not supported" ) - override def getMaxProcedureNameLength: Int = throw new UJESSQLException( - UJESSQLErrorCode.NOSUPPORT_METADATA, + override def getMaxProcedureNameLength: Int = throw new LinkisSQLException( + LinkisSQLErrorCode.NOSUPPORT_METADATA, "getMaxProcedureNameLength not supported" ) - override def getMaxCatalogNameLength: Int = throw new UJESSQLException( - UJESSQLErrorCode.NOSUPPORT_METADATA, + override def getMaxCatalogNameLength: Int = throw new LinkisSQLException( + LinkisSQLErrorCode.NOSUPPORT_METADATA, "getMaxCatalogNameLength not supported" ) override def getMaxRowSize: Int = - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_METADATA, "getMaxRowSize not supported") + throw new LinkisSQLException( + LinkisSQLErrorCode.NOSUPPORT_METADATA, + "getMaxRowSize not supported" + ) - override def doesMaxRowSizeIncludeBlobs(): Boolean = throw new UJESSQLException( - UJESSQLErrorCode.NOSUPPORT_METADATA, + override def doesMaxRowSizeIncludeBlobs(): Boolean = throw new LinkisSQLException( + LinkisSQLErrorCode.NOSUPPORT_METADATA, "doesMaxRowSizeIncludeBlobs not supported" ) - override def getMaxStatementLength: Int = throw new UJESSQLException( - UJESSQLErrorCode.NOSUPPORT_METADATA, + override def getMaxStatementLength: Int = throw new LinkisSQLException( + LinkisSQLErrorCode.NOSUPPORT_METADATA, "getMaxStatementLength not supported" ) - override def getMaxStatements: Int = throw new UJESSQLException( - UJESSQLErrorCode.NOSUPPORT_METADATA, + override def getMaxStatements: Int = throw new LinkisSQLException( + LinkisSQLErrorCode.NOSUPPORT_METADATA, "getMaxStatements not supported" ) - override def getMaxTableNameLength: Int = throw new UJESSQLException( - UJESSQLErrorCode.NOSUPPORT_METADATA, + override def getMaxTableNameLength: Int = throw new LinkisSQLException( + LinkisSQLErrorCode.NOSUPPORT_METADATA, "getMaxTableNameLength not supported" ) - override def getMaxTablesInSelect: Int = throw new UJESSQLException( - UJESSQLErrorCode.NOSUPPORT_METADATA, + override def getMaxTablesInSelect: Int = throw new LinkisSQLException( + LinkisSQLErrorCode.NOSUPPORT_METADATA, "getMaxTablesInSelect not supported" ) - override def getMaxUserNameLength: Int = throw new UJESSQLException( - UJESSQLErrorCode.NOSUPPORT_METADATA, + override def getMaxUserNameLength: Int = throw new LinkisSQLException( + LinkisSQLErrorCode.NOSUPPORT_METADATA, "getMaxUserNameLength not supported" ) @@ -329,13 +336,13 @@ class UJESSQLDatabaseMetaData(ujesSQLConnection: LinkisSQLConnection) override def supportsDataManipulationTransactionsOnly(): Boolean = false - override def dataDefinitionCausesTransactionCommit(): Boolean = throw new UJESSQLException( - UJESSQLErrorCode.NOSUPPORT_METADATA, + override def dataDefinitionCausesTransactionCommit(): Boolean = throw new LinkisSQLException( + LinkisSQLErrorCode.NOSUPPORT_METADATA, "dataDefinitionCausesTransactionCommit not supported" ) - override def dataDefinitionIgnoredInTransactions(): Boolean = throw new UJESSQLException( - UJESSQLErrorCode.NOSUPPORT_METADATA, + override def dataDefinitionIgnoredInTransactions(): Boolean = throw new LinkisSQLException( + LinkisSQLErrorCode.NOSUPPORT_METADATA, "dataDefinitionIgnoredInTransactions not supported" ) @@ -679,48 +686,48 @@ class UJESSQLDatabaseMetaData(ujesSQLConnection: LinkisSQLConnection) override def supportsResultSetConcurrency(`type`: Int, concurrency: Int): Boolean = false - override def ownUpdatesAreVisible(`type`: Int): Boolean = throw new UJESSQLException( - UJESSQLErrorCode.NOSUPPORT_METADATA, + override def ownUpdatesAreVisible(`type`: Int): Boolean = throw new LinkisSQLException( + LinkisSQLErrorCode.NOSUPPORT_METADATA, "ownUpdatesAreVisible not supported" ) - override def ownDeletesAreVisible(`type`: Int): Boolean = throw new UJESSQLException( - UJESSQLErrorCode.NOSUPPORT_METADATA, + override def ownDeletesAreVisible(`type`: Int): Boolean = throw new LinkisSQLException( + LinkisSQLErrorCode.NOSUPPORT_METADATA, "ownDeletesAreVisible not supported" ) - override def ownInsertsAreVisible(`type`: Int): Boolean = throw new UJESSQLException( - UJESSQLErrorCode.NOSUPPORT_METADATA, + override def ownInsertsAreVisible(`type`: Int): Boolean = throw new LinkisSQLException( + LinkisSQLErrorCode.NOSUPPORT_METADATA, "ownInsertsAreVisible not supported" ) - override def othersUpdatesAreVisible(`type`: Int): Boolean = throw new UJESSQLException( - UJESSQLErrorCode.NOSUPPORT_METADATA, + override def othersUpdatesAreVisible(`type`: Int): Boolean = throw new LinkisSQLException( + LinkisSQLErrorCode.NOSUPPORT_METADATA, "othersUpdatesAreVisible not supported" ) - override def othersDeletesAreVisible(`type`: Int): Boolean = throw new UJESSQLException( - UJESSQLErrorCode.NOSUPPORT_METADATA, + override def othersDeletesAreVisible(`type`: Int): Boolean = throw new LinkisSQLException( + LinkisSQLErrorCode.NOSUPPORT_METADATA, "othersDeletesAreVisible not supported" ) - override def othersInsertsAreVisible(`type`: Int): Boolean = throw new UJESSQLException( - UJESSQLErrorCode.NOSUPPORT_METADATA, + override def othersInsertsAreVisible(`type`: Int): Boolean = throw new LinkisSQLException( + LinkisSQLErrorCode.NOSUPPORT_METADATA, "othersInsertsAreVisible not supported" ) - override def updatesAreDetected(`type`: Int): Boolean = throw new UJESSQLException( - UJESSQLErrorCode.NOSUPPORT_METADATA, + override def updatesAreDetected(`type`: Int): Boolean = throw new LinkisSQLException( + LinkisSQLErrorCode.NOSUPPORT_METADATA, "updatesAreDetected not supported" ) - override def deletesAreDetected(`type`: Int): Boolean = throw new UJESSQLException( - UJESSQLErrorCode.NOSUPPORT_METADATA, + override def deletesAreDetected(`type`: Int): Boolean = throw new LinkisSQLException( + LinkisSQLErrorCode.NOSUPPORT_METADATA, "deletesAreDetected not supported" ) - override def insertsAreDetected(`type`: Int): Boolean = throw new UJESSQLException( - UJESSQLErrorCode.NOSUPPORT_METADATA, + override def insertsAreDetected(`type`: Int): Boolean = throw new LinkisSQLException( + LinkisSQLErrorCode.NOSUPPORT_METADATA, "insertsAreDetected not supported" ) @@ -748,14 +755,20 @@ class UJESSQLDatabaseMetaData(ujesSQLConnection: LinkisSQLConnection) schemaPattern: String, typeNamePattern: String ): ResultSet = - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_METADATA, "getSuperTypes not supported") + throw new LinkisSQLException( + LinkisSQLErrorCode.NOSUPPORT_METADATA, + "getSuperTypes not supported" + ) override def getSuperTables( catalog: String, schemaPattern: String, tableNamePattern: String ): ResultSet = - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_METADATA, "getSuperTables not supported") + throw new LinkisSQLException( + LinkisSQLErrorCode.NOSUPPORT_METADATA, + "getSuperTables not supported" + ) override def getAttributes( catalog: String, @@ -763,12 +776,15 @@ class UJESSQLDatabaseMetaData(ujesSQLConnection: LinkisSQLConnection) typeNamePattern: String, attributeNamePattern: String ): ResultSet = - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_METADATA, "getAttributes not supported") + throw new LinkisSQLException( + LinkisSQLErrorCode.NOSUPPORT_METADATA, + "getAttributes not supported" + ) override def supportsResultSetHoldability(holdability: Int): Boolean = false - override def getResultSetHoldability: Int = throw new UJESSQLException( - UJESSQLErrorCode.NOSUPPORT_METADATA, + override def getResultSetHoldability: Int = throw new LinkisSQLException( + LinkisSQLErrorCode.NOSUPPORT_METADATA, "getResultSetHoldability not supported" ) @@ -782,15 +798,15 @@ class UJESSQLDatabaseMetaData(ujesSQLConnection: LinkisSQLConnection) override def getSQLStateType: Int = 2 - override def locatorsUpdateCopy(): Boolean = throw new UJESSQLException( - UJESSQLErrorCode.NOSUPPORT_METADATA, + override def locatorsUpdateCopy(): Boolean = throw new LinkisSQLException( + LinkisSQLErrorCode.NOSUPPORT_METADATA, "locatorsUpdateCopy not supported" ) override def supportsStatementPooling(): Boolean = false - override def getRowIdLifetime: RowIdLifetime = throw new UJESSQLException( - UJESSQLErrorCode.NOSUPPORT_METADATA, + override def getRowIdLifetime: RowIdLifetime = throw new LinkisSQLException( + LinkisSQLErrorCode.NOSUPPORT_METADATA, "getRowIdLifetime not supported" ) @@ -803,13 +819,13 @@ class UJESSQLDatabaseMetaData(ujesSQLConnection: LinkisSQLConnection) override def supportsStoredFunctionsUsingCallSyntax(): Boolean = false - override def autoCommitFailureClosesAllResultSets(): Boolean = throw new UJESSQLException( - UJESSQLErrorCode.NOSUPPORT_METADATA, + override def autoCommitFailureClosesAllResultSets(): Boolean = throw new LinkisSQLException( + LinkisSQLErrorCode.NOSUPPORT_METADATA, "autoCommitFailureClosesAllResultSets not supported" ) - override def getClientInfoProperties: ResultSet = throw new UJESSQLException( - UJESSQLErrorCode.NOSUPPORT_METADATA, + override def getClientInfoProperties: ResultSet = throw new LinkisSQLException( + LinkisSQLErrorCode.NOSUPPORT_METADATA, "getClientInfoProperties not supported" ) @@ -833,15 +849,18 @@ class UJESSQLDatabaseMetaData(ujesSQLConnection: LinkisSQLConnection) columnNamePattern: String ): ResultSet = null - override def generatedKeyAlwaysReturned(): Boolean = throw new UJESSQLException( - UJESSQLErrorCode.NOSUPPORT_METADATA, + override def generatedKeyAlwaysReturned(): Boolean = throw new LinkisSQLException( + LinkisSQLErrorCode.NOSUPPORT_METADATA, "generatedKeyAlwaysReturned not supported" ) override def unwrap[T](iface: Class[T]): T = - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_METADATA, "unwrap not supported") + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_METADATA, "unwrap not supported") override def isWrapperFor(iface: Class[_]): Boolean = - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_METADATA, "isWrapperFor not supported") + throw new LinkisSQLException( + LinkisSQLErrorCode.NOSUPPORT_METADATA, + "isWrapperFor not supported" + ) } diff --git a/linkis-computation-governance/linkis-jdbc-driver/src/main/scala/org/apache/linkis/ujes/jdbc/UJESSQLDriverMain.scala b/linkis-computation-governance/linkis-jdbc-driver/src/main/scala/org/apache/linkis/ujes/jdbc/UJESSQLDriverMain.scala index 9fb4c0f605..6296f399e5 100644 --- a/linkis-computation-governance/linkis-jdbc-driver/src/main/scala/org/apache/linkis/ujes/jdbc/UJESSQLDriverMain.scala +++ b/linkis-computation-governance/linkis-jdbc-driver/src/main/scala/org/apache/linkis/ujes/jdbc/UJESSQLDriverMain.scala @@ -78,14 +78,17 @@ class UJESSQLDriverMain extends Driver with Logging { false case Array(key, _) => if (StringUtils.isBlank(key)) { - throw new UJESSQLException(UJESSQLErrorCode.BAD_URL, "bad url for params: " + url) + throw new LinkisSQLException( + LinkisSQLErrorCode.BAD_URL, + "bad url for params: " + url + ) } else true case _ => - throw new UJESSQLException(UJESSQLErrorCode.BAD_URL, "bad url for params: " + url) + throw new LinkisSQLException(LinkisSQLErrorCode.BAD_URL, "bad url for params: " + url) } props.setProperty(PARAMS, kvs.map(_.mkString(KV_SPLIT)).mkString(PARAM_SPLIT)) } - case _ => throw new UJESSQLException(UJESSQLErrorCode.BAD_URL, "bad url: " + url) + case _ => throw new LinkisSQLException(LinkisSQLErrorCode.BAD_URL, "bad url: " + url) } props } diff --git a/linkis-computation-governance/linkis-jdbc-driver/src/main/scala/org/apache/linkis/ujes/jdbc/UJESSQLResultSet.scala b/linkis-computation-governance/linkis-jdbc-driver/src/main/scala/org/apache/linkis/ujes/jdbc/UJESSQLResultSet.scala index 97bfc576e3..5df56733e4 100644 --- a/linkis-computation-governance/linkis-jdbc-driver/src/main/scala/org/apache/linkis/ujes/jdbc/UJESSQLResultSet.scala +++ b/linkis-computation-governance/linkis-jdbc-driver/src/main/scala/org/apache/linkis/ujes/jdbc/UJESSQLResultSet.scala @@ -241,7 +241,7 @@ class UJESSQLResultSet( } } else { dataType.toLowerCase(Locale.getDefault) match { - case null => throw new UJESSQLException(UJESSQLErrorCode.METADATA_EMPTY) + case null => throw new LinkisSQLException(LinkisSQLErrorCode.METADATA_EMPTY) case "string" => value.toString case "short" => value.toShort case "int" => value.toInt @@ -260,8 +260,8 @@ class UJESSQLResultSet( case "array" => value.toArray case "map" => value case _ => - throw new UJESSQLException( - UJESSQLErrorCode.PREPARESTATEMENT_TYPEERROR, + throw new LinkisSQLException( + LinkisSQLErrorCode.PREPARESTATEMENT_TYPEERROR, s"Can't infer the SQL type to use for an instance of ${dataType}. Use getObject() with an explicit Types value to specify the type to use" ) } @@ -270,15 +270,15 @@ class UJESSQLResultSet( private def getColumnValue(columnIndex: Int): Any = { if (currentRow == null) { - throw new UJESSQLException(UJESSQLErrorCode.RESULTSET_ROWERROR, "No row found.") + throw new LinkisSQLException(LinkisSQLErrorCode.RESULTSET_ROWERROR, "No row found.") } else if (currentRow.size() <= 0) { - throw new UJESSQLException( - UJESSQLErrorCode.RESULTSET_ROWERROR, + throw new LinkisSQLException( + LinkisSQLErrorCode.RESULTSET_ROWERROR, "RowSet does not contain any columns!" ) } else if (columnIndex > currentRow.size()) { - throw new UJESSQLException( - UJESSQLErrorCode.RESULTSET_ROWERROR, + throw new LinkisSQLException( + LinkisSQLErrorCode.RESULTSET_ROWERROR, s" Invalid columnIndex: ${columnIndex}" ) } else { @@ -293,7 +293,7 @@ class UJESSQLResultSet( override def getString(columnIndex: Int): String = { val any = getColumnValue(columnIndex) if (wasNull()) { - throw new UJESSQLException(UJESSQLErrorCode.RESULTSET_ROWERROR, "Type is null") + throw new LinkisSQLException(LinkisSQLErrorCode.RESULTSET_ROWERROR, "Type is null") } else { any match { case c: Character => Character.toString(c) @@ -305,7 +305,7 @@ class UJESSQLResultSet( override def getBoolean(columnIndex: Int): Boolean = { val any = getColumnValue(columnIndex) if (wasNull()) { - throw new UJESSQLException(UJESSQLErrorCode.RESULTSET_ROWERROR, "Type is null") + throw new LinkisSQLException(LinkisSQLErrorCode.RESULTSET_ROWERROR, "Type is null") } else { any match { case s: String => @@ -321,7 +321,7 @@ class UJESSQLResultSet( override def getByte(columnIndex: Int): Byte = { val any = getColumnValue(columnIndex) if (wasNull()) { - throw new UJESSQLException(UJESSQLErrorCode.RESULTSET_ROWERROR, "Type is null") + throw new LinkisSQLException(LinkisSQLErrorCode.RESULTSET_ROWERROR, "Type is null") } else { any.asInstanceOf[Byte] } @@ -330,7 +330,7 @@ class UJESSQLResultSet( override def getShort(columnIndex: Int): Short = { val any = getColumnValue(columnIndex) if (wasNull()) { - throw new UJESSQLException(UJESSQLErrorCode.RESULTSET_ROWERROR, "Type is null") + throw new LinkisSQLException(LinkisSQLErrorCode.RESULTSET_ROWERROR, "Type is null") } else { any.asInstanceOf[Short] } @@ -339,7 +339,7 @@ class UJESSQLResultSet( override def getInt(columnIndex: Int): Int = { val any = getColumnValue(columnIndex) if (wasNull()) { - throw new UJESSQLException(UJESSQLErrorCode.RESULTSET_ROWERROR, "Type is null") + throw new LinkisSQLException(LinkisSQLErrorCode.RESULTSET_ROWERROR, "Type is null") } else { any match { case i: Integer => i.asInstanceOf[Int] @@ -352,7 +352,7 @@ class UJESSQLResultSet( override def getLong(columnIndex: Int): Long = { val any = getColumnValue(columnIndex) if (wasNull()) { - throw new UJESSQLException(UJESSQLErrorCode.RESULTSET_ROWERROR, "Type is null") + throw new LinkisSQLException(LinkisSQLErrorCode.RESULTSET_ROWERROR, "Type is null") } else { any match { case i: Integer => i.longValue() @@ -365,7 +365,7 @@ class UJESSQLResultSet( override def getFloat(columnIndex: Int): Float = { val any = getColumnValue(columnIndex) if (wasNull()) { - throw new UJESSQLException(UJESSQLErrorCode.RESULTSET_ROWERROR, "Type is null") + throw new LinkisSQLException(LinkisSQLErrorCode.RESULTSET_ROWERROR, "Type is null") } else { any.asInstanceOf[Float] } @@ -374,7 +374,7 @@ class UJESSQLResultSet( override def getDouble(columnIndex: Int): Double = { val any = getColumnValue(columnIndex) if (wasNull()) { - throw new UJESSQLException(UJESSQLErrorCode.RESULTSET_ROWERROR, "Type is null") + throw new LinkisSQLException(LinkisSQLErrorCode.RESULTSET_ROWERROR, "Type is null") } else { any match { case _: String => 0.0d @@ -387,7 +387,7 @@ class UJESSQLResultSet( val mc = new MathContext(scale) val any = getColumnValue(columnIndex) if (wasNull()) { - throw new UJESSQLException(UJESSQLErrorCode.RESULTSET_ROWERROR, "Type is null") + throw new LinkisSQLException(LinkisSQLErrorCode.RESULTSET_ROWERROR, "Type is null") } else { any match { case double: Double => new java.math.BigDecimal(double).round(mc) @@ -401,7 +401,7 @@ class UJESSQLResultSet( override def getBytes(columnIndex: Int): Array[Byte] = { val any = getColumnValue(columnIndex) if (wasNull()) { - throw new UJESSQLException(UJESSQLErrorCode.RESULTSET_ROWERROR, "Type is null") + throw new LinkisSQLException(LinkisSQLErrorCode.RESULTSET_ROWERROR, "Type is null") } else { any.asInstanceOf[Array[Byte]] } @@ -412,35 +412,35 @@ class UJESSQLResultSet( val any = getColumnValue(columnIndex) logger.info(s"the value of Date is $any") if (wasNull()) { - throw new UJESSQLException(UJESSQLErrorCode.RESULTSET_ROWERROR, "Type is null") + throw new LinkisSQLException(LinkisSQLErrorCode.RESULTSET_ROWERROR, "Type is null") } else { any.asInstanceOf[Date] } } override def getTime(columnIndex: Int): Time = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_RESULTSET) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_RESULTSET) } override def getTimestamp(columnIndex: Int): Timestamp = { val any = getColumnValue(columnIndex) if (wasNull()) { - throw new UJESSQLException(UJESSQLErrorCode.RESULTSET_ROWERROR, "Type is null") + throw new LinkisSQLException(LinkisSQLErrorCode.RESULTSET_ROWERROR, "Type is null") } else { any.asInstanceOf[Timestamp] } } override def getAsciiStream(columnIndex: Int): InputStream = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_RESULTSET) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_RESULTSET) } override def getUnicodeStream(columnIndex: Int): InputStream = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_RESULTSET) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_RESULTSET) } override def getBinaryStream(columnIndex: Int): InputStream = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_RESULTSET) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_RESULTSET) } override def getString(columnLabel: String): String = { @@ -488,7 +488,7 @@ class UJESSQLResultSet( } override def getTime(columnLabel: String): Time = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_RESULTSET) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_RESULTSET) } override def getTimestamp(columnLabel: String): Timestamp = { @@ -496,15 +496,15 @@ class UJESSQLResultSet( } override def getAsciiStream(columnLabel: String): InputStream = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_RESULTSET) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_RESULTSET) } override def getUnicodeStream(columnLabel: String): InputStream = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_RESULTSET) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_RESULTSET) } override def getBinaryStream(columnLabel: String): InputStream = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_RESULTSET) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_RESULTSET) } override def getWarnings: SQLWarning = { @@ -516,7 +516,7 @@ class UJESSQLResultSet( } override def getCursorName: String = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_RESULTSET) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_RESULTSET) } override def getMetaData: UJESSQLResultSetMetaData = { @@ -546,19 +546,19 @@ class UJESSQLResultSet( } } if (columnIndex == -1) { - throw new UJESSQLException( - UJESSQLErrorCode.RESULTSET_ROWERROR, + throw new LinkisSQLException( + LinkisSQLErrorCode.RESULTSET_ROWERROR, s"can not find column: ${columnLabel}" ) } else columnIndex } override def getCharacterStream(columnIndex: Int): Reader = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_RESULTSET) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_RESULTSET) } override def getCharacterStream(columnLabel: String): Reader = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_RESULTSET) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_RESULTSET) } override def getBigDecimal(columnIndex: Int): java.math.BigDecimal = { @@ -571,31 +571,31 @@ class UJESSQLResultSet( override def isBeforeFirst: Boolean = { if (resultSetRow == null) { - throw new UJESSQLException(UJESSQLErrorCode.RESULTSET_NULL) + throw new LinkisSQLException(LinkisSQLErrorCode.RESULTSET_NULL) } else currentRowCursor == -1 } override def isAfterLast: Boolean = { if (resultSetRow == null) { - throw new UJESSQLException(UJESSQLErrorCode.RESULTSET_NULL) + throw new LinkisSQLException(LinkisSQLErrorCode.RESULTSET_NULL) } else currentRowCursor > resultSetRow.size() - 1 } override def isFirst: Boolean = { if (resultSetRow == null) { - throw new UJESSQLException(UJESSQLErrorCode.RESULTSET_NULL) + throw new LinkisSQLException(LinkisSQLErrorCode.RESULTSET_NULL) } else currentRowCursor == 0 } override def isLast: Boolean = { if (resultSetRow == null) { - throw new UJESSQLException(UJESSQLErrorCode.RESULTSET_NULL) + throw new LinkisSQLException(LinkisSQLErrorCode.RESULTSET_NULL) } else currentRowCursor == resultSetRow.size() - 1 } override def beforeFirst(): Unit = { if (resultSetRow == null) { - throw new UJESSQLException(UJESSQLErrorCode.RESULTSET_NULL) + throw new LinkisSQLException(LinkisSQLErrorCode.RESULTSET_NULL) } else { currentRowCursor = -1 updateCurrentRow(currentRowCursor) @@ -604,7 +604,7 @@ class UJESSQLResultSet( override def afterLast(): Unit = { if (resultSetRow == null) { - throw new UJESSQLException(UJESSQLErrorCode.RESULTSET_NULL) + throw new LinkisSQLException(LinkisSQLErrorCode.RESULTSET_NULL) } else { currentRowCursor = resultSetRow.size() updateCurrentRow(currentRowCursor) @@ -631,7 +631,7 @@ class UJESSQLResultSet( override def getRow: Int = { if (resultSetRow == null) { - throw new UJESSQLException(UJESSQLErrorCode.RESULTSET_NULL) + throw new LinkisSQLException(LinkisSQLErrorCode.RESULTSET_NULL) } else { currentRowCursor + 1 } @@ -639,10 +639,10 @@ class UJESSQLResultSet( override def absolute(row: Int): Boolean = { if (resultSetRow == null) { - throw new UJESSQLException(UJESSQLErrorCode.RESULTSET_NULL) + throw new LinkisSQLException(LinkisSQLErrorCode.RESULTSET_NULL) } else if (row > resultSetRow.size()) { - throw new UJESSQLException( - UJESSQLErrorCode.RESULTSET_ROWERROR, + throw new LinkisSQLException( + LinkisSQLErrorCode.RESULTSET_ROWERROR, "The specified number of rows is greater than the maximum number of rows" ) } else { @@ -658,10 +658,10 @@ class UJESSQLResultSet( override def relative(rows: Int): Boolean = { if (resultSetRow == null) { - throw new UJESSQLException(UJESSQLErrorCode.RESULTSET_NULL) + throw new LinkisSQLException(LinkisSQLErrorCode.RESULTSET_NULL) } else if (rows > resultSetRow.size()) { - throw new UJESSQLException( - UJESSQLErrorCode.RESULTSET_ROWERROR, + throw new LinkisSQLException( + LinkisSQLErrorCode.RESULTSET_ROWERROR, "The specified number of rows is greater than the maximum number of rows" ) } else { @@ -683,11 +683,11 @@ class UJESSQLResultSet( } override def setFetchDirection(direction: Int): Unit = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_RESULTSET) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_RESULTSET) } override def getFetchDirection: Int = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_RESULTSET) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_RESULTSET) } override def setFetchSize(rows: Int): Unit = { @@ -707,248 +707,248 @@ class UJESSQLResultSet( } override def rowUpdated(): Boolean = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_RESULTSET) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_RESULTSET) } override def rowInserted(): Boolean = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_RESULTSET) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_RESULTSET) } override def rowDeleted(): Boolean = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_RESULTSET) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_RESULTSET) } override def updateNull(columnIndex: Int): Unit = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_RESULTSET) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_RESULTSET) } override def updateBoolean(columnIndex: Int, x: Boolean): Unit = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_RESULTSET) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_RESULTSET) } override def updateByte(columnIndex: Int, x: Byte): Unit = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_RESULTSET) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_RESULTSET) } override def updateShort(columnIndex: Int, x: Short): Unit = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_RESULTSET) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_RESULTSET) } override def updateInt(columnIndex: Int, x: Int): Unit = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_RESULTSET) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_RESULTSET) } override def updateLong(columnIndex: Int, x: Long): Unit = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_RESULTSET) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_RESULTSET) } override def updateFloat(columnIndex: Int, x: Float): Unit = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_RESULTSET) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_RESULTSET) } override def updateDouble(columnIndex: Int, x: Double): Unit = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_RESULTSET) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_RESULTSET) } override def updateBigDecimal(columnIndex: Int, x: java.math.BigDecimal): Unit = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_RESULTSET) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_RESULTSET) } override def updateString(columnIndex: Int, x: String): Unit = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_RESULTSET) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_RESULTSET) } override def updateBytes(columnIndex: Int, x: Array[Byte]): Unit = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_RESULTSET) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_RESULTSET) } override def updateDate(columnIndex: Int, x: Date): Unit = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_RESULTSET) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_RESULTSET) } override def updateTime(columnIndex: Int, x: Time): Unit = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_RESULTSET) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_RESULTSET) } override def updateTimestamp(columnIndex: Int, x: Timestamp): Unit = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_RESULTSET) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_RESULTSET) } override def updateAsciiStream(columnIndex: Int, x: InputStream, length: Int): Unit = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_RESULTSET) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_RESULTSET) } override def updateBinaryStream(columnIndex: Int, x: InputStream, length: Int): Unit = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_RESULTSET) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_RESULTSET) } override def updateCharacterStream(columnIndex: Int, x: Reader, length: Int): Unit = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_RESULTSET) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_RESULTSET) } override def updateObject(columnIndex: Int, x: scala.Any, scaleOrLength: Int): Unit = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_RESULTSET) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_RESULTSET) } override def updateObject(columnIndex: Int, x: scala.Any): Unit = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_RESULTSET) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_RESULTSET) } override def updateNull(columnLabel: String): Unit = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_RESULTSET) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_RESULTSET) } override def updateBoolean(columnLabel: String, x: Boolean): Unit = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_RESULTSET) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_RESULTSET) } override def updateByte(columnLabel: String, x: Byte): Unit = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_RESULTSET) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_RESULTSET) } override def updateShort(columnLabel: String, x: Short): Unit = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_RESULTSET) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_RESULTSET) } override def updateInt(columnLabel: String, x: Int): Unit = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_RESULTSET) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_RESULTSET) } override def updateLong(columnLabel: String, x: Long): Unit = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_RESULTSET) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_RESULTSET) } override def updateFloat(columnLabel: String, x: Float): Unit = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_RESULTSET) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_RESULTSET) } override def updateDouble(columnLabel: String, x: Double): Unit = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_RESULTSET) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_RESULTSET) } override def updateBigDecimal(columnLabel: String, x: java.math.BigDecimal): Unit = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_RESULTSET) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_RESULTSET) } override def updateString(columnLabel: String, x: String): Unit = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_RESULTSET) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_RESULTSET) } override def updateBytes(columnLabel: String, x: Array[Byte]): Unit = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_RESULTSET) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_RESULTSET) } override def updateDate(columnLabel: String, x: Date): Unit = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_RESULTSET) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_RESULTSET) } override def updateTime(columnLabel: String, x: Time): Unit = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_RESULTSET) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_RESULTSET) } override def updateTimestamp(columnLabel: String, x: Timestamp): Unit = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_RESULTSET) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_RESULTSET) } override def updateAsciiStream(columnLabel: String, x: InputStream, length: Int): Unit = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_RESULTSET) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_RESULTSET) } override def updateBinaryStream(columnLabel: String, x: InputStream, length: Int): Unit = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_RESULTSET) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_RESULTSET) } override def updateCharacterStream(columnLabel: String, reader: Reader, length: Int): Unit = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_RESULTSET) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_RESULTSET) } override def updateObject(columnLabel: String, x: scala.Any, scaleOrLength: Int): Unit = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_RESULTSET) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_RESULTSET) } override def updateObject(columnLabel: String, x: scala.Any): Unit = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_RESULTSET) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_RESULTSET) } override def insertRow(): Unit = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_RESULTSET) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_RESULTSET) } override def updateRow(): Unit = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_RESULTSET) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_RESULTSET) } override def deleteRow(): Unit = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_RESULTSET) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_RESULTSET) } override def refreshRow(): Unit = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_RESULTSET) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_RESULTSET) } override def cancelRowUpdates(): Unit = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_RESULTSET) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_RESULTSET) } override def moveToInsertRow(): Unit = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_RESULTSET) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_RESULTSET) } override def moveToCurrentRow(): Unit = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_RESULTSET) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_RESULTSET) } override def getStatement: Statement = { if (statement != null && !hasClosed) { statement.asInstanceOf[Statement] - } else throw new UJESSQLException(UJESSQLErrorCode.STATEMENT_CLOSED) + } else throw new LinkisSQLException(LinkisSQLErrorCode.STATEMENT_CLOSED) } override def getObject(columnIndex: Int, map: util.Map[String, Class[_]]): AnyRef = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_RESULTSET) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_RESULTSET) } override def getRef(columnIndex: Int): Ref = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_RESULTSET) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_RESULTSET) } override def getBlob(columnIndex: Int): Blob = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_RESULTSET) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_RESULTSET) } override def getClob(columnIndex: Int): Clob = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_RESULTSET) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_RESULTSET) } override def getArray(columnIndex: Int): sql.Array = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_RESULTSET) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_RESULTSET) } override def getObject(columnLabel: String, map: util.Map[String, Class[_]]): AnyRef = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_RESULTSET) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_RESULTSET) } override def getRef(columnLabel: String): Ref = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_RESULTSET) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_RESULTSET) } override def getBlob(columnLabel: String): Blob = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_RESULTSET) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_RESULTSET) } override def getClob(columnLabel: String): Clob = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_RESULTSET) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_RESULTSET) } override def getArray(columnLabel: String): sql.Array = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_RESULTSET) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_RESULTSET) } private def getDate(columnIndex: Int, localTimeZone: DateTimeZone): Date = { val value = getColumnValue(columnIndex) logger.info(s"the value of value is $value and the value of localTimeZone is $localTimeZone") if (wasNull()) { - throw new UJESSQLException(UJESSQLErrorCode.RESULTSET_ROWERROR, "Type is null") + throw new LinkisSQLException(LinkisSQLErrorCode.RESULTSET_ROWERROR, "Type is null") } else new Date(DATE_FORMATTER.withZone(localTimeZone).parseMillis(String.valueOf(value))); } @@ -962,11 +962,11 @@ class UJESSQLResultSet( } override def getTime(columnIndex: Int, cal: Calendar): Time = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_RESULTSET) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_RESULTSET) } override def getTime(columnLabel: String, cal: Calendar): Time = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_RESULTSET) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_RESULTSET) } private def getTimestamp(columnIndex: Int, localTimeZone: DateTimeZone): Timestamp = { @@ -990,63 +990,63 @@ class UJESSQLResultSet( } override def getURL(columnIndex: Int): URL = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_RESULTSET) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_RESULTSET) } override def getURL(columnLabel: String): URL = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_RESULTSET) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_RESULTSET) } override def updateRef(columnIndex: Int, x: Ref): Unit = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_RESULTSET) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_RESULTSET) } override def updateRef(columnLabel: String, x: Ref): Unit = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_RESULTSET) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_RESULTSET) } override def updateBlob(columnIndex: Int, x: Blob): Unit = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_RESULTSET) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_RESULTSET) } override def updateBlob(columnLabel: String, x: Blob): Unit = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_RESULTSET) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_RESULTSET) } override def updateClob(columnIndex: Int, x: Clob): Unit = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_RESULTSET) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_RESULTSET) } override def updateClob(columnLabel: String, x: Clob): Unit = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_RESULTSET) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_RESULTSET) } override def updateArray(columnIndex: Int, x: sql.Array): Unit = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_RESULTSET) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_RESULTSET) } override def updateArray(columnLabel: String, x: sql.Array): Unit = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_RESULTSET) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_RESULTSET) } override def getRowId(columnIndex: Int): RowId = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_RESULTSET) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_RESULTSET) } override def getRowId(columnLabel: String): RowId = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_RESULTSET) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_RESULTSET) } override def updateRowId(columnIndex: Int, x: RowId): Unit = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_RESULTSET) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_RESULTSET) } override def updateRowId(columnLabel: String, x: RowId): Unit = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_RESULTSET) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_RESULTSET) } override def getHoldability: Int = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_RESULTSET) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_RESULTSET) } override def isClosed: Boolean = { @@ -1054,187 +1054,187 @@ class UJESSQLResultSet( } override def updateNString(columnIndex: Int, nString: String): Unit = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_RESULTSET) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_RESULTSET) } override def updateNString(columnLabel: String, nString: String): Unit = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_RESULTSET) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_RESULTSET) } override def updateNClob(columnIndex: Int, nClob: NClob): Unit = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_RESULTSET) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_RESULTSET) } override def updateNClob(columnLabel: String, nClob: NClob): Unit = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_RESULTSET) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_RESULTSET) } override def getNClob(columnIndex: Int): NClob = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_RESULTSET) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_RESULTSET) } override def getNClob(columnLabel: String): NClob = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_RESULTSET) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_RESULTSET) } override def getSQLXML(columnIndex: Int): SQLXML = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_RESULTSET) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_RESULTSET) } override def getSQLXML(columnLabel: String): SQLXML = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_RESULTSET) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_RESULTSET) } override def updateSQLXML(columnIndex: Int, xmlObject: SQLXML): Unit = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_RESULTSET) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_RESULTSET) } override def updateSQLXML(columnLabel: String, xmlObject: SQLXML): Unit = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_RESULTSET) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_RESULTSET) } override def getNString(columnIndex: Int): String = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_RESULTSET) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_RESULTSET) } override def getNString(columnLabel: String): String = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_RESULTSET) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_RESULTSET) } override def getNCharacterStream(columnIndex: Int): Reader = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_RESULTSET) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_RESULTSET) } override def getNCharacterStream(columnLabel: String): Reader = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_RESULTSET) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_RESULTSET) } override def updateNCharacterStream(columnIndex: Int, x: Reader, length: Long): Unit = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_RESULTSET) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_RESULTSET) } override def updateNCharacterStream(columnLabel: String, reader: Reader, length: Long): Unit = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_RESULTSET) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_RESULTSET) } override def updateAsciiStream(columnIndex: Int, x: InputStream, length: Long): Unit = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_RESULTSET) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_RESULTSET) } override def updateBinaryStream(columnIndex: Int, x: InputStream, length: Long): Unit = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_RESULTSET) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_RESULTSET) } override def updateCharacterStream(columnIndex: Int, x: Reader, length: Long): Unit = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_RESULTSET) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_RESULTSET) } override def updateAsciiStream(columnLabel: String, x: InputStream, length: Long): Unit = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_RESULTSET) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_RESULTSET) } override def updateBinaryStream(columnLabel: String, x: InputStream, length: Long): Unit = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_RESULTSET) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_RESULTSET) } override def updateCharacterStream(columnLabel: String, reader: Reader, length: Long): Unit = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_RESULTSET) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_RESULTSET) } override def updateBlob(columnIndex: Int, inputStream: InputStream, length: Long): Unit = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_RESULTSET) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_RESULTSET) } override def updateBlob(columnLabel: String, inputStream: InputStream, length: Long): Unit = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_RESULTSET) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_RESULTSET) } override def updateClob(columnIndex: Int, reader: Reader, length: Long): Unit = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_RESULTSET) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_RESULTSET) } override def updateClob(columnLabel: String, reader: Reader, length: Long): Unit = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_RESULTSET) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_RESULTSET) } override def updateNClob(columnIndex: Int, reader: Reader, length: Long): Unit = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_RESULTSET) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_RESULTSET) } override def updateNClob(columnLabel: String, reader: Reader, length: Long): Unit = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_RESULTSET) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_RESULTSET) } override def updateNCharacterStream(columnIndex: Int, x: Reader): Unit = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_RESULTSET) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_RESULTSET) } override def updateNCharacterStream(columnLabel: String, reader: Reader): Unit = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_RESULTSET) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_RESULTSET) } override def updateAsciiStream(columnIndex: Int, x: InputStream): Unit = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_RESULTSET) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_RESULTSET) } override def updateBinaryStream(columnIndex: Int, x: InputStream): Unit = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_RESULTSET) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_RESULTSET) } override def updateCharacterStream(columnIndex: Int, x: Reader): Unit = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_RESULTSET) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_RESULTSET) } override def updateAsciiStream(columnLabel: String, x: InputStream): Unit = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_RESULTSET) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_RESULTSET) } override def updateBinaryStream(columnLabel: String, x: InputStream): Unit = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_RESULTSET) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_RESULTSET) } override def updateCharacterStream(columnLabel: String, reader: Reader): Unit = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_RESULTSET) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_RESULTSET) } override def updateBlob(columnIndex: Int, inputStream: InputStream): Unit = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_RESULTSET) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_RESULTSET) } override def updateBlob(columnLabel: String, inputStream: InputStream): Unit = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_RESULTSET) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_RESULTSET) } override def updateClob(columnIndex: Int, reader: Reader): Unit = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_RESULTSET) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_RESULTSET) } override def updateClob(columnLabel: String, reader: Reader): Unit = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_RESULTSET) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_RESULTSET) } override def updateNClob(columnIndex: Int, reader: Reader): Unit = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_RESULTSET) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_RESULTSET) } override def updateNClob(columnLabel: String, reader: Reader): Unit = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_RESULTSET) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_RESULTSET) } override def getObject[T](columnIndex: Int, `type`: Class[T]): T = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_RESULTSET) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_RESULTSET) } override def getObject[T](columnLabel: String, `type`: Class[T]): T = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_RESULTSET) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_RESULTSET) } override def unwrap[T](iface: Class[T]): T = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_RESULTSET) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_RESULTSET) } override def isWrapperFor(iface: Class[_]): Boolean = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_RESULTSET) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_RESULTSET) } } diff --git a/linkis-computation-governance/linkis-jdbc-driver/src/main/scala/org/apache/linkis/ujes/jdbc/UJESSQLResultSetMetaData.scala b/linkis-computation-governance/linkis-jdbc-driver/src/main/scala/org/apache/linkis/ujes/jdbc/UJESSQLResultSetMetaData.scala index e51309e885..eaeb1b25d8 100644 --- a/linkis-computation-governance/linkis-jdbc-driver/src/main/scala/org/apache/linkis/ujes/jdbc/UJESSQLResultSetMetaData.scala +++ b/linkis-computation-governance/linkis-jdbc-driver/src/main/scala/org/apache/linkis/ujes/jdbc/UJESSQLResultSetMetaData.scala @@ -33,19 +33,19 @@ class UJESSQLResultSetMetaData extends ResultSetMetaData with Logging { private[jdbc] def setColumnNameProperties(column: Int, columnName: String): Unit = { if (column != null && columnName != null) { columnNameProperties.put(column, columnName) - } else throw new UJESSQLException(UJESSQLErrorCode.METADATA_EMPTY) + } else throw new LinkisSQLException(LinkisSQLErrorCode.METADATA_EMPTY) } private[jdbc] def setDataTypeProperties(column: Int, columnName: String): Unit = { if (column != null && columnName != null) { dataTypeProperties.put(column, columnName) - } else throw new UJESSQLException(UJESSQLErrorCode.METADATA_EMPTY) + } else throw new LinkisSQLException(LinkisSQLErrorCode.METADATA_EMPTY) } private[jdbc] def setCommentPropreties(column: Int, columnName: String): Unit = { if (column != null && columnName != null) { commentProperties.put(column, columnName) - } else throw new UJESSQLException(UJESSQLErrorCode.METADATA_EMPTY) + } else throw new LinkisSQLException(LinkisSQLErrorCode.METADATA_EMPTY) } override def getColumnCount: Int = { @@ -53,7 +53,7 @@ class UJESSQLResultSetMetaData extends ResultSetMetaData with Logging { } override def isAutoIncrement(column: Int): Boolean = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_METADATA) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_METADATA) } override def isCaseSensitive(column: Int): Boolean = true @@ -74,7 +74,7 @@ class UJESSQLResultSetMetaData extends ResultSetMetaData with Logging { override def getColumnLabel(column: Int): String = { if (columnNameProperties.get(column) == null) { - throw new UJESSQLException(UJESSQLErrorCode.METADATA_EMPTY) + throw new LinkisSQLException(LinkisSQLErrorCode.METADATA_EMPTY) } else columnNameProperties.get(column) } @@ -83,7 +83,7 @@ class UJESSQLResultSetMetaData extends ResultSetMetaData with Logging { } override def getSchemaName(column: Int): String = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_METADATA) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_METADATA) } override def getPrecision(column: Int): Int = { @@ -102,11 +102,11 @@ class UJESSQLResultSetMetaData extends ResultSetMetaData with Logging { } override def getTableName(column: Int): String = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_METADATA) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_METADATA) } override def getCatalogName(column: Int): String = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_METADATA) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_METADATA) } override def getColumnType(column: Int): Int = { @@ -115,7 +115,7 @@ class UJESSQLResultSetMetaData extends ResultSetMetaData with Logging { override def getColumnTypeName(column: Int): String = { if (dataTypeProperties.get(column) == null) { - throw new UJESSQLException(UJESSQLErrorCode.METADATA_EMPTY) + throw new LinkisSQLException(LinkisSQLErrorCode.METADATA_EMPTY) } else dataTypeProperties.get(column) } @@ -128,19 +128,19 @@ class UJESSQLResultSetMetaData extends ResultSetMetaData with Logging { } override def isDefinitelyWritable(column: Int): Boolean = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_METADATA) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_METADATA) } override def getColumnClassName(column: Int): String = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_METADATA) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_METADATA) } override def unwrap[T](iface: Class[T]): T = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_METADATA) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_METADATA) } override def isWrapperFor(iface: Class[_]): Boolean = { - throw new UJESSQLException(UJESSQLErrorCode.NOSUPPORT_METADATA) + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_METADATA) } protected def toZeroIndex(column: Int): Int = { diff --git a/linkis-computation-governance/linkis-jdbc-driver/src/main/scala/org/apache/linkis/ujes/jdbc/UJESSQLTypeParser.scala b/linkis-computation-governance/linkis-jdbc-driver/src/main/scala/org/apache/linkis/ujes/jdbc/UJESSQLTypeParser.scala index 541064b2d5..d8de812a1b 100644 --- a/linkis-computation-governance/linkis-jdbc-driver/src/main/scala/org/apache/linkis/ujes/jdbc/UJESSQLTypeParser.scala +++ b/linkis-computation-governance/linkis-jdbc-driver/src/main/scala/org/apache/linkis/ujes/jdbc/UJESSQLTypeParser.scala @@ -25,7 +25,7 @@ object UJESSQLTypeParser { def parserFromName(typeName: String): Int = { val typeNameLowerCase = typeName.toLowerCase(Locale.getDefault()) typeName.toLowerCase() match { - case null => throw new UJESSQLException(UJESSQLErrorCode.METADATA_EMPTY) + case null => throw new LinkisSQLException(LinkisSQLErrorCode.METADATA_EMPTY) case "string" => Types.NVARCHAR case "short" => Types.SMALLINT case "int" => Types.INTEGER @@ -67,7 +67,7 @@ object UJESSQLTypeParser { case _: Char => Types.CHAR case _: BigDecimal => Types.DECIMAL case _: Timestamp => Types.TIMESTAMP - case _ => throw new UJESSQLException(UJESSQLErrorCode.PREPARESTATEMENT_TYPEERROR) + case _ => throw new LinkisSQLException(LinkisSQLErrorCode.PREPARESTATEMENT_TYPEERROR) } } @@ -87,7 +87,7 @@ object UJESSQLTypeParser { case Types.VARCHAR => "varchar" case Types.NVARCHAR => "string" case Types.DATE => "date" - case _ => throw new UJESSQLException(UJESSQLErrorCode.PREPARESTATEMENT_TYPEERROR) + case _ => throw new LinkisSQLException(LinkisSQLErrorCode.PREPARESTATEMENT_TYPEERROR) } } diff --git a/linkis-computation-governance/linkis-jdbc-driver/src/test/java/org/apache/linkis/ujes/jdbc/UJESSQLDatabaseMetaDataTest.java b/linkis-computation-governance/linkis-jdbc-driver/src/test/java/org/apache/linkis/ujes/jdbc/UJESSQLDatabaseMetaDataTest.java index e7c6ade996..100e13f3e4 100644 --- a/linkis-computation-governance/linkis-jdbc-driver/src/test/java/org/apache/linkis/ujes/jdbc/UJESSQLDatabaseMetaDataTest.java +++ b/linkis-computation-governance/linkis-jdbc-driver/src/test/java/org/apache/linkis/ujes/jdbc/UJESSQLDatabaseMetaDataTest.java @@ -54,42 +54,43 @@ public static void preWork() { @Test public void supportsMinimumSQLGrammar() { if (dbmd != null) { - Assertions.assertThrows(UJESSQLException.class, () -> dbmd.supportsMinimumSQLGrammar()); + Assertions.assertThrows(LinkisSQLException.class, () -> dbmd.supportsMinimumSQLGrammar()); } } @Test public void getResultSetHoldability() { if (dbmd != null) { - Assertions.assertThrows(UJESSQLException.class, () -> dbmd.getResultSetHoldability()); + Assertions.assertThrows(LinkisSQLException.class, () -> dbmd.getResultSetHoldability()); } } @Test public void getMaxColumnsInGroupBy() { if (dbmd != null) { - Assertions.assertThrows(UJESSQLException.class, () -> dbmd.getMaxColumnsInGroupBy()); + Assertions.assertThrows(LinkisSQLException.class, () -> dbmd.getMaxColumnsInGroupBy()); } } @Test public void supportsSubqueriesInComparisons() { if (dbmd != null) { - Assertions.assertThrows(UJESSQLException.class, () -> dbmd.supportsSubqueriesInComparisons()); + Assertions.assertThrows( + LinkisSQLException.class, () -> dbmd.supportsSubqueriesInComparisons()); } } @Test public void getMaxColumnsInSelect() { if (dbmd != null) { - Assertions.assertThrows(UJESSQLException.class, () -> dbmd.getMaxColumnsInSelect()); + Assertions.assertThrows(LinkisSQLException.class, () -> dbmd.getMaxColumnsInSelect()); } } @Test public void nullPlusNonNullIsNull() { if (dbmd != null) { - Assertions.assertThrows(UJESSQLException.class, () -> dbmd.nullPlusNonNullIsNull()); + Assertions.assertThrows(LinkisSQLException.class, () -> dbmd.nullPlusNonNullIsNull()); } } @@ -104,7 +105,7 @@ public void supportsCatalogsInDataManipulation() { public void supportsDataDefinitionAndDataManipulationTransactions() { if (dbmd != null) { Assertions.assertThrows( - UJESSQLException.class, + LinkisSQLException.class, () -> dbmd.supportsDataDefinitionAndDataManipulationTransactions()); } } @@ -112,7 +113,7 @@ public void supportsDataDefinitionAndDataManipulationTransactions() { @Test public void supportsTableCorrelationNames() { if (dbmd != null) { - Assertions.assertThrows(UJESSQLException.class, () -> dbmd.supportsTableCorrelationNames()); + Assertions.assertThrows(LinkisSQLException.class, () -> dbmd.supportsTableCorrelationNames()); } } @@ -133,7 +134,7 @@ public void supportsFullOuterJoins() { @Test public void supportsExpressionsInOrderBy() { if (dbmd != null) { - Assertions.assertThrows(UJESSQLException.class, () -> dbmd.supportsExpressionsInOrderBy()); + Assertions.assertThrows(LinkisSQLException.class, () -> dbmd.supportsExpressionsInOrderBy()); } } @@ -147,14 +148,14 @@ public void allProceduresAreCallable() { @Test public void getMaxTablesInSelect() { if (dbmd != null) { - Assertions.assertThrows(UJESSQLException.class, () -> dbmd.getMaxTablesInSelect()); + Assertions.assertThrows(LinkisSQLException.class, () -> dbmd.getMaxTablesInSelect()); } } @Test public void nullsAreSortedAtStart() { if (dbmd != null) { - Assertions.assertThrows(UJESSQLException.class, () -> dbmd.nullsAreSortedAtStart()); + Assertions.assertThrows(LinkisSQLException.class, () -> dbmd.nullsAreSortedAtStart()); } } @@ -168,7 +169,7 @@ public void supportsPositionedUpdate() { @Test public void ownDeletesAreVisible() { if (dbmd != null) { - Assertions.assertThrows(UJESSQLException.class, () -> dbmd.ownDeletesAreVisible(0)); + Assertions.assertThrows(LinkisSQLException.class, () -> dbmd.ownDeletesAreVisible(0)); } } @@ -182,14 +183,14 @@ public void supportsResultSetHoldability() { @Test public void getMaxStatements() { if (dbmd != null) { - Assertions.assertThrows(UJESSQLException.class, () -> dbmd.getMaxStatements()); + Assertions.assertThrows(LinkisSQLException.class, () -> dbmd.getMaxStatements()); } } @Test public void getRowIdLifetime() { if (dbmd != null) { - Assertions.assertThrows(UJESSQLException.class, () -> dbmd.getRowIdLifetime()); + Assertions.assertThrows(LinkisSQLException.class, () -> dbmd.getRowIdLifetime()); } } From 8ad47b772ace361cdfd89d471fc8bef5238bd668 Mon Sep 17 00:00:00 2001 From: peacewong Date: Mon, 15 May 2023 20:46:39 +0800 Subject: [PATCH 109/261] Fix build error --- .../org/apache/linkis/ujes/jdbc/LinkisSQLStatement.scala | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/linkis-computation-governance/linkis-jdbc-driver/src/main/scala/org/apache/linkis/ujes/jdbc/LinkisSQLStatement.scala b/linkis-computation-governance/linkis-jdbc-driver/src/main/scala/org/apache/linkis/ujes/jdbc/LinkisSQLStatement.scala index 88cf3013ca..7ae2321f65 100644 --- a/linkis-computation-governance/linkis-jdbc-driver/src/main/scala/org/apache/linkis/ujes/jdbc/LinkisSQLStatement.scala +++ b/linkis-computation-governance/linkis-jdbc-driver/src/main/scala/org/apache/linkis/ujes/jdbc/LinkisSQLStatement.scala @@ -22,13 +22,11 @@ import org.apache.linkis.governance.common.entity.ExecutionNodeStatus import org.apache.linkis.ujes.client.request.OpenLogAction import org.apache.linkis.ujes.client.response.JobExecuteResult import org.apache.linkis.ujes.jdbc.hook.JDBCDriverPreExecutionHook - import org.apache.commons.lang3.StringUtils -import java.sql._ +import java.sql.{Connection, ResultSet, SQLException, SQLWarning, Statement} import java.util import java.util.concurrent.TimeUnit - import scala.concurrent.TimeoutException import scala.concurrent.duration.Duration From 30abd1ba8810f3d46836f9e80042517c143ac470 Mon Sep 17 00:00:00 2001 From: huangKai-2323 <62878639+huangKai-2323@users.noreply.github.com> Date: Mon, 15 May 2023 23:44:52 +0800 Subject: [PATCH 110/261] Dev 1.1.11 optimize udf permissions (#154) * Code optimization * add udf function createtime && updatetime column --- linkis-dist/package/db/linkis_ddl.sql | 2 +- .../package/db/upgrade/1.4.0_schema/mysql/linkis_ddl.sql | 2 +- .../basedatamanager/server/restful/UdfManagerRestfulApi.java | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/linkis-dist/package/db/linkis_ddl.sql b/linkis-dist/package/db/linkis_ddl.sql index b4df979f84..94b94a8c32 100644 --- a/linkis-dist/package/db/linkis_ddl.sql +++ b/linkis-dist/package/db/linkis_ddl.sql @@ -197,7 +197,7 @@ CREATE TABLE `linkis_ps_udf_tree` ( `update_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, `category` varchar(50) DEFAULT NULL COMMENT 'Used to distinguish between udf and function', PRIMARY KEY (`id`), - UNIQUE KEY `uniq_name_uname_category` (`name`, `user_name`, `category`) + UNIQUE KEY `uniq_parent_name_uname_category` (`parent`,`name`,`user_name`,`category`) ) ENGINE=InnoDB DEFAULT CHARSET=utf8; diff --git a/linkis-dist/package/db/upgrade/1.4.0_schema/mysql/linkis_ddl.sql b/linkis-dist/package/db/upgrade/1.4.0_schema/mysql/linkis_ddl.sql index 8de7cf7632..c3da20e590 100644 --- a/linkis-dist/package/db/upgrade/1.4.0_schema/mysql/linkis_ddl.sql +++ b/linkis-dist/package/db/upgrade/1.4.0_schema/mysql/linkis_ddl.sql @@ -30,4 +30,4 @@ ALTER TABLE `linkis_ps_udf_user_load` ADD COLUMN `update_time` datetime NOT NUL ALTER TABLE `linkis_ps_udf_version` ADD COLUMN `update_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'update_time'; -ALTER TABLE `linkis_ps_udf_tree` ADD CONSTRAINT `uniq_name_uname_category` UNIQUE (`name`, `user_name`, `category`); \ No newline at end of file +ALTER TABLE `linkis_ps_udf_tree` ADD CONSTRAINT `uniq_parent_name_uname_category` UNIQUE (`parent`,`name`, `user_name`, `category`); \ No newline at end of file diff --git a/linkis-public-enhancements/linkis-basedata-manager/src/main/java/org/apache/linkis/basedatamanager/server/restful/UdfManagerRestfulApi.java b/linkis-public-enhancements/linkis-basedata-manager/src/main/java/org/apache/linkis/basedatamanager/server/restful/UdfManagerRestfulApi.java index e56e1bda48..588b0b6a26 100644 --- a/linkis-public-enhancements/linkis-basedata-manager/src/main/java/org/apache/linkis/basedatamanager/server/restful/UdfManagerRestfulApi.java +++ b/linkis-public-enhancements/linkis-basedata-manager/src/main/java/org/apache/linkis/basedatamanager/server/restful/UdfManagerRestfulApi.java @@ -130,8 +130,8 @@ public Message update( if (!Configuration.isAdmin(username)) { return Message.error("User '" + username + "' is not admin user[非管理员用户]"); } - QueryWrapper queryWrapper = - new QueryWrapper<>(udfManagerEntity).eq("user_name", udfManagerEntity.getUserName()); + QueryWrapper queryWrapper = new QueryWrapper(); + queryWrapper.eq("user_name", udfManagerEntity.getUserName()); UdfManagerEntity udfManager = udfManagerService.getOne(queryWrapper); if (udfManager == null) { udfManagerEntity.setUpdateTime(new Date()); From 2cf371c6a4cb1471c68f7703b4064af13fb27227 Mon Sep 17 00:00:00 2001 From: peacewong Date: Tue, 16 May 2023 21:02:45 +0800 Subject: [PATCH 111/261] Code Format --- linkis-computation-governance/linkis-jdbc-driver/pom.xml | 9 +++++++++ .../org/apache/linkis/ujes/jdbc/LinkisSQLStatement.scala | 2 ++ 2 files changed, 11 insertions(+) diff --git a/linkis-computation-governance/linkis-jdbc-driver/pom.xml b/linkis-computation-governance/linkis-jdbc-driver/pom.xml index cdf23c5f21..28437d95a5 100644 --- a/linkis-computation-governance/linkis-jdbc-driver/pom.xml +++ b/linkis-computation-governance/linkis-jdbc-driver/pom.xml @@ -41,6 +41,15 @@ + + + org.apache.maven.plugins + maven-surefire-plugin + + true + + + net.alchim31.maven scala-maven-plugin diff --git a/linkis-computation-governance/linkis-jdbc-driver/src/main/scala/org/apache/linkis/ujes/jdbc/LinkisSQLStatement.scala b/linkis-computation-governance/linkis-jdbc-driver/src/main/scala/org/apache/linkis/ujes/jdbc/LinkisSQLStatement.scala index 7ae2321f65..f00d870978 100644 --- a/linkis-computation-governance/linkis-jdbc-driver/src/main/scala/org/apache/linkis/ujes/jdbc/LinkisSQLStatement.scala +++ b/linkis-computation-governance/linkis-jdbc-driver/src/main/scala/org/apache/linkis/ujes/jdbc/LinkisSQLStatement.scala @@ -22,11 +22,13 @@ import org.apache.linkis.governance.common.entity.ExecutionNodeStatus import org.apache.linkis.ujes.client.request.OpenLogAction import org.apache.linkis.ujes.client.response.JobExecuteResult import org.apache.linkis.ujes.jdbc.hook.JDBCDriverPreExecutionHook + import org.apache.commons.lang3.StringUtils import java.sql.{Connection, ResultSet, SQLException, SQLWarning, Statement} import java.util import java.util.concurrent.TimeUnit + import scala.concurrent.TimeoutException import scala.concurrent.duration.Duration From b64c1a3d5df27d6b7a2f4dc6085faad689889d82 Mon Sep 17 00:00:00 2001 From: Yonghao Mei <73584269+mayinrain@users.noreply.github.com> Date: Wed, 17 May 2023 19:31:13 +0800 Subject: [PATCH 112/261] fix web bugs and optimization (#156) --- .../apps/linkis/assets/styles/console.scss | 7 +- .../src/apps/linkis/i18n/common/en.json | 5 +- .../src/apps/linkis/i18n/common/zh.json | 5 +- .../src/apps/linkis/module/ECM/engineConn.vue | 2 +- .../src/apps/linkis/module/ECM/index.vue | 2 +- .../linkis/module/datasourceAccess/index.vue | 3 +- .../module/datasourceEnv/EditForm/index.vue | 15 +-- .../linkis/module/datasourceEnv/index.vue | 11 +- .../linkis/module/datasourceType/index.vue | 3 +- .../linkis/module/datasourceTypeKey/index.vue | 2 +- .../apps/linkis/module/errorCode/index.vue | 2 +- .../linkis/module/gatewayAuthToken/index.vue | 2 +- .../module/globalHistoryManagement/index.vue | 2 +- .../module/microServiceManagement/index.vue | 2 +- .../module/resourceManagement/engineConn.vue | 2 +- .../rmExternalResourceProvider/index.vue | 2 +- .../module/tenantTagManagement/index.vue | 2 +- .../apps/linkis/module/udfManager/index.vue | 2 +- .../linkis/module/udfTree/EditForm/index.vue | 109 ++++++++++++++++-- .../src/apps/linkis/module/udfTree/index.scss | 1 - .../src/apps/linkis/module/udfTree/index.vue | 23 ++-- .../src/apps/linkis/module/udfTree/service.js | 4 +- .../src/apps/linkis/view/linkis/index.vue | 2 +- 23 files changed, 159 insertions(+), 51 deletions(-) diff --git a/linkis-web/src/apps/linkis/assets/styles/console.scss b/linkis-web/src/apps/linkis/assets/styles/console.scss index 4f89988448..45a7a4a6d7 100644 --- a/linkis-web/src/apps/linkis/assets/styles/console.scss +++ b/linkis-web/src/apps/linkis/assets/styles/console.scss @@ -15,7 +15,9 @@ * limitations under the License. */ -@import '@/common/style/variables.scss'; + @charset "UTF-8"; + @import '@/common/style/variables.scss'; + .console-page{ position: $relative; width: $percent-all; @@ -45,6 +47,9 @@ } } .content-body-side-right { + .ivu-page-item-jump-next:after, .ivu-page-item-jump-prev:after { + content: "\2022\2022\2022"; + } flex: 1; background: $body-background; border-radius: $border-radius-small; diff --git a/linkis-web/src/apps/linkis/i18n/common/en.json b/linkis-web/src/apps/linkis/i18n/common/en.json index aba5ff0c61..c2a692250f 100644 --- a/linkis-web/src/apps/linkis/i18n/common/en.json +++ b/linkis-web/src/apps/linkis/i18n/common/en.json @@ -90,6 +90,7 @@ "linkText": "View result set", "largeResultTips": "The front end displays only 5000 pieces of data" }, + "needPre": "Need to select category and input username before", "warning": { "api": "Requesting API, please hold on!", "data": "Requesting data, please hold on!", @@ -188,7 +189,8 @@ "datasourceTypeKey": "Datasource Config Options", "EnginePluginManagement": "Engine Plugin Management", "basedataManagement": "Basic Data Management", - "codeQuery": "Code Retrieval" + "codeQuery": "Code Retrieval", + "EngineConnList": "Engine Conn List" } } }, @@ -463,6 +465,7 @@ "searchPlaceholder": "Please enter your search", "search": "Search", "action": "Action", + "categoryMaxLength": "The length cannot exceed 100", "modal":{ "confirm": "Confirm", "cancel": "Cancel", diff --git a/linkis-web/src/apps/linkis/i18n/common/zh.json b/linkis-web/src/apps/linkis/i18n/common/zh.json index 49950288ca..9e9182ea18 100644 --- a/linkis-web/src/apps/linkis/i18n/common/zh.json +++ b/linkis-web/src/apps/linkis/i18n/common/zh.json @@ -90,6 +90,7 @@ "linkText": "点击查看结果集", "largeResultTips": "前端只展示5000条数据" }, + "needPre": "需要先选择分类和输入用户名", "warning": { "api": "接口请求中，请稍候！", "data": "数据请求中，请稍候！", @@ -188,7 +189,8 @@ "EnginePluginManagement": "引擎物料管理", "basedataManagement": "基础数据管理", "codeQuery": "代码检索", - "engineConfigurationTemplate": "引擎配置模板" + "engineConfigurationTemplate": "引擎配置模板", + "EngineConnList": "引擎列表" } } }, @@ -441,6 +443,7 @@ "searchPlaceholder": "请输入搜索内容", "search": "搜索", "action": "操作", + "categoryMaxLength": "分类名长度不能超过100", "modal":{ "confirm": "确定", "cancel": "取消", diff --git a/linkis-web/src/apps/linkis/module/ECM/engineConn.vue b/linkis-web/src/apps/linkis/module/ECM/engineConn.vue index 5fd9c221cc..3f19ff3a84 100644 --- a/linkis-web/src/apps/linkis/module/ECM/engineConn.vue +++ b/linkis-web/src/apps/linkis/module/ECM/engineConn.vue @@ -472,7 +472,7 @@ export default { instance: this.instance, } }, - emInstance: e.instance, + emInstance: e.instance.replace(/ /g, ''), nodeStatus: e.nodeHealthy, owner: e.owner, engineType: e.engineType diff --git a/linkis-web/src/apps/linkis/module/ECM/index.vue b/linkis-web/src/apps/linkis/module/ECM/index.vue index f38588c8f8..4794cdf8f3 100644 --- a/linkis-web/src/apps/linkis/module/ECM/index.vue +++ b/linkis-web/src/apps/linkis/module/ECM/index.vue @@ -402,7 +402,7 @@ export default { // search(搜索) search(e) { let param = { - instance: e.instance, + instance: e.instance.replace(/ /g, ''), nodeHealthy: e.nodeHealthy, owner: e.owner, tenantLabel: e.tenant, diff --git a/linkis-web/src/apps/linkis/module/datasourceAccess/index.vue b/linkis-web/src/apps/linkis/module/datasourceAccess/index.vue index 33f30de9bb..f0fda1cb48 100644 --- a/linkis-web/src/apps/linkis/module/datasourceAccess/index.vue +++ b/linkis-web/src/apps/linkis/module/datasourceAccess/index.vue @@ -219,6 +219,8 @@ export default { duration: 3, content: "删除成功" }) + + this.load() }else{ this.$Message.success({ duration: 3, @@ -226,7 +228,6 @@ export default { }) } }) - this.load() } }) diff --git a/linkis-web/src/apps/linkis/module/datasourceEnv/EditForm/index.vue b/linkis-web/src/apps/linkis/module/datasourceEnv/EditForm/index.vue index ca19bd336c..6d7838d59d 100644 --- a/linkis-web/src/apps/linkis/module/datasourceEnv/EditForm/index.vue +++ b/linkis-web/src/apps/linkis/module/datasourceEnv/EditForm/index.vue @@ -118,7 +118,7 @@ export default { ], on: { 'on-change': () => { - this.rule[8].hidden = !this.rule[8].hidden; + this.rule[8].hidden = !this.rule[4].value; } } }, @@ -199,7 +199,7 @@ export default { title: "principle", field: 'principle', value: '', - hidden: true, + hidden: false, props: { placeholder: 'hadoop@APACHE.COM', }, @@ -208,7 +208,7 @@ export default { type: 'v-jsoneditor', title: this.$t('message.linkis.basedataManagement.datasourceEnv.parameter'), field: 'hadoopConf', - value: {}, + value: JSON.stringify({}), props: { type: 'form-create', height: "280px", @@ -251,12 +251,9 @@ export default { // keytab value this.rule[6].hidden = !newV.keytab; // upload - this.rule[5].hidden = !this.rule[6].hidden; - if (!newV.hasKeyTab || !['hive', 'kafka'].includes(dataSourceTypeName)){ - this.rule[5].hidden = true; - this.rule[6].hidden = true; - this.rule[8].hidden = true; - } + this.rule[5].hidden = !this.rule[6].hidden || !['hive', 'kafka'].includes(dataSourceTypeName) || !this.rule[4].value; + this.rule[8].hidden = (!newV.hasKeyTab || !['hive', 'kafka'].includes(dataSourceTypeName)); + }, keyTabValidator(rule, val, cb) { if (!this.formData.keytab) { diff --git a/linkis-web/src/apps/linkis/module/datasourceEnv/index.vue b/linkis-web/src/apps/linkis/module/datasourceEnv/index.vue index 553817e8c5..854d4d3435 100644 --- a/linkis-web/src/apps/linkis/module/datasourceEnv/index.vue +++ b/linkis-web/src/apps/linkis/module/datasourceEnv/index.vue @@ -239,6 +239,8 @@ export default { this.modalShow = true }, onTableEdit(row){ + // console.log(this.$refs.editForm); + this.$refs.editForm.formModel.reload() row.hasKeyTab = JSON.parse(row.parameter).keytab ? true : false; this.modalEditData = {...row} // format parameter for modal @@ -267,6 +269,8 @@ export default { duration: 3, content: this.$t('message.linkis.basedataManagement.modal.modalDeleteSuccess') }) + + this.load() }else{ this.$Message.success({ duration: 3, @@ -274,7 +278,6 @@ export default { }) } }) - this.load() } }) @@ -282,9 +285,9 @@ export default { clearForm(){ for(let key in this.modalEditData) { this.modalEditData[key] = '' - this.modalEditData.parameter = {} - window.console.log(key); } + this.modalEditData.parameter = {} + this.modalEditData.hadoopConf = JSON.stringify({}) this.modalEditData.hasKeyTab = false; }, async onModalOk(){ @@ -345,7 +348,7 @@ export default { } }) } - window.console.log(formData); + // window.console.log(formData); this.modalLoading=false this.modalShow = false }) diff --git a/linkis-web/src/apps/linkis/module/datasourceType/index.vue b/linkis-web/src/apps/linkis/module/datasourceType/index.vue index 5dbb8bf6f7..8d8e5b0770 100644 --- a/linkis-web/src/apps/linkis/module/datasourceType/index.vue +++ b/linkis-web/src/apps/linkis/module/datasourceType/index.vue @@ -221,6 +221,8 @@ export default { duration: 3, content: this.$t('message.linkis.basedataManagement.modal.modalDeleteSuccess') }) + + this.load() }else{ this.$Message.success({ duration: 3, @@ -228,7 +230,6 @@ export default { }) } }) - this.load() } }) diff --git a/linkis-web/src/apps/linkis/module/datasourceTypeKey/index.vue b/linkis-web/src/apps/linkis/module/datasourceTypeKey/index.vue index 124faa352e..d47eef8d09 100644 --- a/linkis-web/src/apps/linkis/module/datasourceTypeKey/index.vue +++ b/linkis-web/src/apps/linkis/module/datasourceTypeKey/index.vue @@ -270,13 +270,13 @@ export default { duration: 3, content: this.$t('message.linkis.basedataManagement.modal.modalDeleteSuccess') }) + this.load() }else{ this.$Message.success({ duration: 3, content: this.$t('message.linkis.basedataManagement.modal.modalDeleteFail') }) } - this.load() }) } }) diff --git a/linkis-web/src/apps/linkis/module/errorCode/index.vue b/linkis-web/src/apps/linkis/module/errorCode/index.vue index cd04394995..b274cc2d35 100644 --- a/linkis-web/src/apps/linkis/module/errorCode/index.vue +++ b/linkis-web/src/apps/linkis/module/errorCode/index.vue @@ -199,13 +199,13 @@ export default { duration: 3, content: this.$t('message.linkis.basedataManagement.modal.modalDeleteSuccess') }) + this.load() }else{ this.$Message.success({ duration: 3, content: this.$t('message.linkis.basedataManagement.modal.modalDeleteFail') }) } - this.load() }) } }) diff --git a/linkis-web/src/apps/linkis/module/gatewayAuthToken/index.vue b/linkis-web/src/apps/linkis/module/gatewayAuthToken/index.vue index 91a3eaaf00..19fb808ea0 100644 --- a/linkis-web/src/apps/linkis/module/gatewayAuthToken/index.vue +++ b/linkis-web/src/apps/linkis/module/gatewayAuthToken/index.vue @@ -245,13 +245,13 @@ export default { duration: 3, content: this.$t('message.linkis.basedataManagement.modal.modalDeleteSuccess') }) + this.load() }else{ this.$Message.success({ duration: 3, content: this.$t('message.linkis.basedataManagement.modal.modalDeleteFail') }) } - this.load() }) } }) diff --git a/linkis-web/src/apps/linkis/module/globalHistoryManagement/index.vue b/linkis-web/src/apps/linkis/module/globalHistoryManagement/index.vue index 929fd62c2a..8a35fa8695 100644 --- a/linkis-web/src/apps/linkis/module/globalHistoryManagement/index.vue +++ b/linkis-web/src/apps/linkis/module/globalHistoryManagement/index.vue @@ -393,7 +393,7 @@ export default { pageSize: this.pageSetting.pageSize, proxyUser: this.searchBar.proxyUser, isAdminView: this.isAdminModel, - instance: this.searchBar.instance + instance: this.searchBar.instance?.replace(/ /g, '') || '' } if (!this.isAdminModel) { delete params.proxyUser diff --git a/linkis-web/src/apps/linkis/module/microServiceManagement/index.vue b/linkis-web/src/apps/linkis/module/microServiceManagement/index.vue index 71ff29f369..74addbbc56 100644 --- a/linkis-web/src/apps/linkis/module/microServiceManagement/index.vue +++ b/linkis-web/src/apps/linkis/module/microServiceManagement/index.vue @@ -253,7 +253,7 @@ export default { }); }, searchAction() { - this.search(this.instance, this.applicationName); + this.search(this.instance?.replace(/ /g, '') || '', this.applicationName); }, //Jump to Microservices(跳转到微服务) handleTabsJump() { diff --git a/linkis-web/src/apps/linkis/module/resourceManagement/engineConn.vue b/linkis-web/src/apps/linkis/module/resourceManagement/engineConn.vue index f8da125f7c..25b87391f4 100644 --- a/linkis-web/src/apps/linkis/module/resourceManagement/engineConn.vue +++ b/linkis-web/src/apps/linkis/module/resourceManagement/engineConn.vue @@ -330,7 +330,7 @@ export default { // search(搜索) search(e) { let url = '/linkisManager/ecinfo/ecrHistoryList?'; - if (e.instance) url += `instance=${e.instance}&` + if (e.instance) url += `instance=${e.instance?.replace(/ /g, '') || ''}&` if (e.owner) url += `creator=${e.owner}&` if (e.shortcut[0]) url += `startDate=${moment(new Date(e.shortcut[0])).format('YYYY-MM-DD HH:mm:ss')}&` if (e.shortcut[1]) { diff --git a/linkis-web/src/apps/linkis/module/rmExternalResourceProvider/index.vue b/linkis-web/src/apps/linkis/module/rmExternalResourceProvider/index.vue index 9e2f175e95..ed84a8e11b 100644 --- a/linkis-web/src/apps/linkis/module/rmExternalResourceProvider/index.vue +++ b/linkis-web/src/apps/linkis/module/rmExternalResourceProvider/index.vue @@ -207,6 +207,7 @@ export default { duration: 3, content: this.$t('message.linkis.basedataManagement.modal.modalDeleteSuccess') }) + this.load() }else{ this.$Message.success({ duration: 3, @@ -214,7 +215,6 @@ export default { }) } }) - this.load() } }) diff --git a/linkis-web/src/apps/linkis/module/tenantTagManagement/index.vue b/linkis-web/src/apps/linkis/module/tenantTagManagement/index.vue index a7aab77405..75aea5c91b 100644 --- a/linkis-web/src/apps/linkis/module/tenantTagManagement/index.vue +++ b/linkis-web/src/apps/linkis/module/tenantTagManagement/index.vue @@ -328,7 +328,7 @@ export default { if (res.exist) { this.$Message.error(this.$t('message.linkis.tenantTagManagement.userIsExisted')) } - this.tagIsExist = !res.exist; + this.tagIsExist = res.exist; }) this.isRequesting = false } catch (err) { diff --git a/linkis-web/src/apps/linkis/module/udfManager/index.vue b/linkis-web/src/apps/linkis/module/udfManager/index.vue index b1f425acbe..483d261fd0 100644 --- a/linkis-web/src/apps/linkis/module/udfManager/index.vue +++ b/linkis-web/src/apps/linkis/module/udfManager/index.vue @@ -188,13 +188,13 @@ export default { duration: 3, content: this.$t('message.linkis.basedataManagement.modal.modalDeleteSuccess') }) + this.load() }else{ this.$Message.success({ duration: 3, content: this.$t('message.linkis.basedataManagement.modal.modalDeleteFail') }) } - this.load() }) } }) diff --git a/linkis-web/src/apps/linkis/module/udfTree/EditForm/index.vue b/linkis-web/src/apps/linkis/module/udfTree/EditForm/index.vue index fb4f2e2c07..382bebe97d 100644 --- a/linkis-web/src/apps/linkis/module/udfTree/EditForm/index.vue +++ b/linkis-web/src/apps/linkis/module/udfTree/EditForm/index.vue @@ -21,13 +21,14 @@ :rule="rule" v-model="formModel" :option="options" + :value.sync="formData" + ref="formRef" /> + diff --git a/linkis-web/src/apps/linkis/module/udfTree/index.scss b/linkis-web/src/apps/linkis/module/udfTree/index.scss index 7ea4fd680f..f520cdfcdc 100644 --- a/linkis-web/src/apps/linkis/module/udfTree/index.scss +++ b/linkis-web/src/apps/linkis/module/udfTree/index.scss @@ -36,7 +36,6 @@ } } - .table-content { margin-top: 25px; } diff --git a/linkis-web/src/apps/linkis/module/udfTree/index.vue b/linkis-web/src/apps/linkis/module/udfTree/index.vue index 214ed2fff0..5966cd2086 100644 --- a/linkis-web/src/apps/linkis/module/udfTree/index.vue +++ b/linkis-web/src/apps/linkis/module/udfTree/index.vue @@ -83,7 +83,8 @@ - + @@ -210,22 +211,30 @@ export default { this.page.pageNow = value this.load() }, - onAdd(){ + async onAdd(){ + + this.modalEditData = {} + this.modalShow = true + await this.$nextTick() this.$refs.errorCodeForm.formModel.resetFields() + this.$refs.errorCodeForm.formModel.setValue({}) this.modalAddMode = 'add' - this.modalShow = true }, - onTableEdit(row){ + async onTableEdit(row){ + + this.modalEditData = row + this.modalShow = true + await this.$nextTick() + this.$refs.errorCodeForm.formModel.resetFields() this.$refs.errorCodeForm.formModel.setValue(row) this.modalAddMode = 'edit' - this.modalShow = true }, onTableDelete(row){ this.$Modal.confirm({ title: this.$t('message.linkis.basedataManagement.modal.modalTitle'), content: this.$t('message.linkis.basedataManagement.modal.modalDelete', {name: row.name}), - onOk: ()=>{ + onOk: async ()=>{ let params = { id: row.id } @@ -235,6 +244,7 @@ export default { duration: 3, content: this.$t('message.linkis.basedataManagement.modal.modalDeleteSuccess') }) + this.load() }else{ this.$Message.success({ duration: 3, @@ -242,7 +252,6 @@ export default { }) } }) - this.load() } }) diff --git a/linkis-web/src/apps/linkis/module/udfTree/service.js b/linkis-web/src/apps/linkis/module/udfTree/service.js index 2f1400afbc..f4d7ea2b7a 100644 --- a/linkis-web/src/apps/linkis/module/udfTree/service.js +++ b/linkis-web/src/apps/linkis/module/udfTree/service.js @@ -23,8 +23,8 @@ const getList = (params)=> { return api.fetch(udfTreeBaseUrl, params , 'get') } -const getAll = ()=> { - return api.fetch(udfTreeBaseUrl+"/all", 'get') +const getAll = (params)=> { + return api.fetch(udfTreeBaseUrl+"/all", params, 'get') } diff --git a/linkis-web/src/apps/linkis/view/linkis/index.vue b/linkis-web/src/apps/linkis/view/linkis/index.vue index 6749d85cdc..085632906b 100644 --- a/linkis-web/src/apps/linkis/view/linkis/index.vue +++ b/linkis-web/src/apps/linkis/view/linkis/index.vue @@ -76,7 +76,7 @@ Date: Wed, 17 May 2023 20:57:48 +0800 Subject: [PATCH 113/261] Dev 1.1.11 webank fixoom (#157) --- .../storage/exception/StorageErrorCode.java | 4 +++- .../resultset/StorageResultSetReader.scala | 24 ++++++++++++++----- .../executor/cs/CSResourceParser.scala | 19 ++++++++------- 3 files changed, 32 insertions(+), 15 deletions(-) diff --git a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/exception/StorageErrorCode.java b/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/exception/StorageErrorCode.java index 3c82ceb523..de6d0eda01 100644 --- a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/exception/StorageErrorCode.java +++ b/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/exception/StorageErrorCode.java @@ -20,7 +20,9 @@ public enum StorageErrorCode { /** */ - FS_NOT_INIT(53001, "please init first(请先初始化)"); + FS_NOT_INIT(53001, "please init first(请先初始化)"), + + FS_OOM(53002, "OOM occurred while reading the file"); StorageErrorCode(int errorCode, String message) { this.code = errorCode; diff --git a/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/resultset/StorageResultSetReader.scala b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/resultset/StorageResultSetReader.scala index 3f49faf3ed..1707867fff 100644 --- a/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/resultset/StorageResultSetReader.scala +++ b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/resultset/StorageResultSetReader.scala @@ -21,7 +21,11 @@ import org.apache.linkis.common.io.{Fs, MetaData, Record} import org.apache.linkis.common.io.resultset.{ResultSet, ResultSetReader} import org.apache.linkis.common.utils.{Logging, Utils} import org.apache.linkis.storage.domain.Dolphin -import org.apache.linkis.storage.exception.StorageWarnException +import org.apache.linkis.storage.exception.{ + StorageErrorCode, + StorageErrorException, + StorageWarnException +} import org.apache.linkis.storage.utils.StorageUtils import java.io.{ByteArrayInputStream, InputStream, IOException} @@ -43,7 +47,6 @@ class StorageResultSetReader[K <: MetaData, V <: Record]( private var fs: Fs = _ private val READ_CACHE = 1024 - private val bytes = new Array[Byte](READ_CACHE) def this(resultSet: ResultSet[K, V], value: String) = { this(resultSet, new ByteArrayInputStream(value.getBytes(Dolphin.CHAR_SET))) @@ -74,9 +77,19 @@ class StorageResultSetReader[K <: MetaData, V <: Record]( case t: Throwable => throw t } - val rowBuffer = ArrayBuffer[Byte]() + var bytes: Array[Byte] = null + try { + bytes = new Array[Byte](rowLen) + } catch { + case e: OutOfMemoryError => + logger.error("Result set read oom, read size {} Byte", rowLen) + throw new StorageErrorException( + StorageErrorCode.FS_OOM.getCode, + StorageErrorCode.FS_OOM.getMessage, + e + ) + } var len = 0 - // Read the entire line, except for the data of the line length(读取整行，除了行长的数据) while (rowLen > 0 && len >= 0) { if (rowLen > READ_CACHE) { @@ -87,11 +100,10 @@ class StorageResultSetReader[K <: MetaData, V <: Record]( if (len > 0) { rowLen -= len - rowBuffer ++= bytes.slice(0, len) } } rowCount = rowCount + 1 - rowBuffer.toArray + bytes } @scala.throws[IOException] diff --git a/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/cs/CSResourceParser.scala b/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/cs/CSResourceParser.scala index f59adaadef..a239915aca 100644 --- a/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/cs/CSResourceParser.scala +++ b/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/cs/CSResourceParser.scala @@ -17,7 +17,9 @@ package org.apache.linkis.engineconn.computation.executor.cs +import org.apache.linkis.common.utils.Logging import org.apache.linkis.cs.client.service.CSResourceService +import org.apache.linkis.governance.common.utils.GovernanceConstant import org.apache.commons.lang3.StringUtils @@ -27,7 +29,7 @@ import java.util.regex.Pattern import scala.collection.JavaConverters._ import scala.collection.mutable.ArrayBuffer -class CSResourceParser { +class CSResourceParser extends Logging { private val pb = Pattern.compile("cs://[^\\s\"]+[$\\s]{0,1}", Pattern.CASE_INSENSITIVE) @@ -47,7 +49,6 @@ class CSResourceParser { nodeNameStr: String ): String = { - // TODO getBMLResource val bmlResourceList = CSResourceService.getInstance().getUpstreamBMLResource(contextIDValueStr, nodeNameStr) @@ -56,23 +57,25 @@ class CSResourceParser { val preFixNames = new ArrayBuffer[String]() val parsedNames = new ArrayBuffer[String]() + val prefixName = System.currentTimeMillis().toString + "_" preFixResourceNames.foreach { preFixResourceName => val resourceName = preFixResourceName.replace(PREFIX, "").trim val bmlResourceOption = bmlResourceList.asScala.find(_.getDownloadedFileName.equals(resourceName)) if (bmlResourceOption.isDefined) { + val replacementName = prefixName + resourceName val bmlResource = bmlResourceOption.get val map = new util.HashMap[String, Object]() - map.put("resourceId", bmlResource.getResourceId) - map.put("version", bmlResource.getVersion) - map.put("fileName", resourceName) + map.put(GovernanceConstant.TASK_RESOURCE_ID_STR, bmlResource.getResourceId) + map.put(GovernanceConstant.TASK_RESOURCE_VERSION_STR, bmlResource.getVersion) + map.put(GovernanceConstant.TASK_RESOURCE_FILE_NAME_STR, replacementName) parsedResources.add(map) preFixNames.append(preFixResourceName) - parsedNames.append(resourceName) + parsedNames.append(replacementName) + logger.warn(s"Replace cs file from {$preFixResourceName} to {$replacementName}") } - } - props.put("resources", parsedResources) + props.put(GovernanceConstant.TASK_RESOURCES_STR, parsedResources) StringUtils.replaceEach(code, preFixNames.toArray, parsedNames.toArray) } From 9885996165aed58c70871ef96ae2b6a1e2d0b3e7 Mon Sep 17 00:00:00 2001 From: peacewong Date: Thu, 18 May 2023 15:26:14 +0800 Subject: [PATCH 114/261] Fix issue with annotation task hang at the end of Scala code --- .../apache/linkis/governance/common/paser/CodeParser.scala | 5 +++++ .../computation/executor/execute/ComputationExecutor.scala | 5 +++++ .../engineplugin/spark/executor/SparkScalaExecutor.scala | 4 ++-- 3 files changed, 12 insertions(+), 2 deletions(-) diff --git a/linkis-computation-governance/linkis-computation-governance-common/src/main/scala/org/apache/linkis/governance/common/paser/CodeParser.scala b/linkis-computation-governance/linkis-computation-governance-common/src/main/scala/org/apache/linkis/governance/common/paser/CodeParser.scala index 19bd7f9cdb..aa99e7efd1 100644 --- a/linkis-computation-governance/linkis-computation-governance-common/src/main/scala/org/apache/linkis/governance/common/paser/CodeParser.scala +++ b/linkis-computation-governance/linkis-computation-governance-common/src/main/scala/org/apache/linkis/governance/common/paser/CodeParser.scala @@ -86,6 +86,11 @@ abstract class CombinedEngineCodeParser extends CodeParser { } +/** + * Scala is no longer using Parser but instead using EmptyParser. If there is a comment at the end, + * it will cause the task to become stuck + */ +@deprecated class ScalaCodeParser extends SingleCodeParser with Logging { override val codeType: CodeType = CodeType.Scala diff --git a/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/execute/ComputationExecutor.scala b/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/execute/ComputationExecutor.scala index 4571e75039..294b4802c3 100644 --- a/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/execute/ComputationExecutor.scala +++ b/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/execute/ComputationExecutor.scala @@ -249,6 +249,11 @@ abstract class ComputationExecutor(val outputPrintLimit: Int = 1000) succeedTasks.increase() transformTaskStatus(engineConnTask, ExecutionNodeStatus.Succeed) s + case incompleteExecuteResponse: IncompleteExecuteResponse => + ErrorExecuteResponse( + s"The task cannot be an incomplete response ${incompleteExecuteResponse.message}", + null + ) case _ => response } response diff --git a/linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/executor/SparkScalaExecutor.scala b/linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/executor/SparkScalaExecutor.scala index 15907bac04..ebb2cdf4c9 100644 --- a/linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/executor/SparkScalaExecutor.scala +++ b/linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/executor/SparkScalaExecutor.scala @@ -31,7 +31,7 @@ import org.apache.linkis.engineplugin.spark.exception.{ SparkSessionNullException } import org.apache.linkis.engineplugin.spark.utils.EngineUtils -import org.apache.linkis.governance.common.paser.ScalaCodeParser +import org.apache.linkis.governance.common.paser.{EmptyCodeParser, ScalaCodeParser} import org.apache.linkis.scheduler.executer.{ ErrorExecuteResponse, ExecuteResponse, @@ -105,7 +105,7 @@ class SparkScalaExecutor(sparkEngineSession: SparkEngineSession, id: Long) System.setProperty("scala.repl.name.line", ("$line" + this.hashCode).replace('-', '0')) - setCodeParser(new ScalaCodeParser) + setCodeParser(new EmptyCodeParser) if (sparkILoop == null) { synchronized { From 597d07a2a171496b8645027cc3f7dc12f60907c0 Mon Sep 17 00:00:00 2001 From: peacewong Date: Thu, 18 May 2023 20:41:55 +0800 Subject: [PATCH 115/261] add runScheduleIntercept --- .../queue/fifoqueue/FIFOUserConsumer.scala | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/linkis-commons/linkis-scheduler/src/main/scala/org/apache/linkis/scheduler/queue/fifoqueue/FIFOUserConsumer.scala b/linkis-commons/linkis-scheduler/src/main/scala/org/apache/linkis/scheduler/queue/fifoqueue/FIFOUserConsumer.scala index 2a40c2517b..6ce6dc4a54 100644 --- a/linkis-commons/linkis-scheduler/src/main/scala/org/apache/linkis/scheduler/queue/fifoqueue/FIFOUserConsumer.scala +++ b/linkis-commons/linkis-scheduler/src/main/scala/org/apache/linkis/scheduler/queue/fifoqueue/FIFOUserConsumer.scala @@ -91,6 +91,14 @@ class FIFOUserConsumer( protected def askExecutorGap(): Unit = {} + /** + * Task scheduling interception is used to judge the rules of task operation, and to judge other task rules based on Group. For example, Entrance makes Creator-level task judgment. + * Note: Only when the interception condition is met can it return, otherwise the rule will be invalid + */ + protected def runScheduleIntercept(): Unit = { + + } + protected def loop(): Unit = { var isRetryJob = false def getWaitForRetryEvent: Option[SchedulerEvent] = { @@ -110,7 +118,7 @@ class FIFOUserConsumer( if (event.isEmpty) { val completedNums = runningJobs.filter(job => job == null || job.isCompleted) if (completedNums.length < 1) { - Utils.tryQuietly(Thread.sleep(1000)) // TODO 还可以优化，通过实现JobListener进行优化 + Utils.tryQuietly(Thread.sleep(1000)) return } while (event.isEmpty) { @@ -119,7 +127,7 @@ class FIFOUserConsumer( if ( takeEvent.exists(e => Utils.tryCatch(e.turnToScheduled()) { t => - takeEvent.get.asInstanceOf[Job].onFailure("Job状态翻转为Scheduled失败！", t) + takeEvent.get.asInstanceOf[Job].onFailure("Failed to change the job status to Scheduled(Job状态翻转为Scheduled失败)", t) false } ) @@ -174,7 +182,7 @@ class FIFOUserConsumer( ) ) case error: Throwable => - job.onFailure("请求引擎失败，可能是由于后台进程错误!请联系管理员", error) + job.onFailure("Failed to request EngineConn", error) if (job.isWaitForRetry) { logger.warn(s"Ask executor for Job $job failed, wait for the next retry!", error) if (!isRetryJob) putToRunningJobs(job) From 4919bfb8003d4b1968a63279dd77f839a534020b Mon Sep 17 00:00:00 2001 From: peacewong Date: Thu, 18 May 2023 20:51:47 +0800 Subject: [PATCH 116/261] Fix storage error --- .../storage/exception/StorageErrorCode.java | 2 +- .../resultset/StorageResultSetReader.scala | 19 +++++++------------ 2 files changed, 8 insertions(+), 13 deletions(-) diff --git a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/exception/StorageErrorCode.java b/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/exception/StorageErrorCode.java index de6d0eda01..8cd4f3603f 100644 --- a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/exception/StorageErrorCode.java +++ b/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/exception/StorageErrorCode.java @@ -21,7 +21,7 @@ public enum StorageErrorCode { /** */ FS_NOT_INIT(53001, "please init first(请先初始化)"), - + INCONSISTENT_DATA(53001, "Inconsistent row data read,read {0},need rowLen {1}"), FS_OOM(53002, "OOM occurred while reading the file"); StorageErrorCode(int errorCode, String message) { diff --git a/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/resultset/StorageResultSetReader.scala b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/resultset/StorageResultSetReader.scala index 1707867fff..8623046087 100644 --- a/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/resultset/StorageResultSetReader.scala +++ b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/resultset/StorageResultSetReader.scala @@ -29,6 +29,7 @@ import org.apache.linkis.storage.exception.{ import org.apache.linkis.storage.utils.StorageUtils import java.io.{ByteArrayInputStream, InputStream, IOException} +import java.text.MessageFormat import scala.collection.mutable.ArrayBuffer @@ -89,18 +90,12 @@ class StorageResultSetReader[K <: MetaData, V <: Record]( e ) } - var len = 0 - // Read the entire line, except for the data of the line length(读取整行，除了行长的数据) - while (rowLen > 0 && len >= 0) { - if (rowLen > READ_CACHE) { - len = StorageUtils.readBytes(inputStream, bytes, READ_CACHE) - } else { - len = StorageUtils.readBytes(inputStream, bytes, rowLen) - } - - if (len > 0) { - rowLen -= len - } + val len = StorageUtils.readBytes(inputStream, bytes, rowLen) + if (len != rowLen) { + throw new StorageErrorException( + StorageErrorCode.INCONSISTENT_DATA.getCode, + MessageFormat.format(StorageErrorCode.INCONSISTENT_DATA.getMessage, len, rowLen) + ) } rowCount = rowCount + 1 bytes From 71fdbe67d31d0f1b3915f1e783e96c53218ff536 Mon Sep 17 00:00:00 2001 From: peacewong Date: Thu, 18 May 2023 21:18:19 +0800 Subject: [PATCH 117/261] Fix build error --- .../org/apache/linkis/storage/exception/StorageErrorCode.java | 2 +- .../linkis/storage/resultset/StorageResultSetReader.scala | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/exception/StorageErrorCode.java b/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/exception/StorageErrorCode.java index 8cd4f3603f..a8123a1f92 100644 --- a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/exception/StorageErrorCode.java +++ b/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/exception/StorageErrorCode.java @@ -21,7 +21,7 @@ public enum StorageErrorCode { /** */ FS_NOT_INIT(53001, "please init first(请先初始化)"), - INCONSISTENT_DATA(53001, "Inconsistent row data read,read {0},need rowLen {1}"), + INCONSISTENT_DATA(53001, "Inconsistent row data read,read %s,need rowLen %s"), FS_OOM(53002, "OOM occurred while reading the file"); StorageErrorCode(int errorCode, String message) { diff --git a/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/resultset/StorageResultSetReader.scala b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/resultset/StorageResultSetReader.scala index 8623046087..9cd4bd2396 100644 --- a/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/resultset/StorageResultSetReader.scala +++ b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/resultset/StorageResultSetReader.scala @@ -29,7 +29,6 @@ import org.apache.linkis.storage.exception.{ import org.apache.linkis.storage.utils.StorageUtils import java.io.{ByteArrayInputStream, InputStream, IOException} -import java.text.MessageFormat import scala.collection.mutable.ArrayBuffer @@ -94,7 +93,7 @@ class StorageResultSetReader[K <: MetaData, V <: Record]( if (len != rowLen) { throw new StorageErrorException( StorageErrorCode.INCONSISTENT_DATA.getCode, - MessageFormat.format(StorageErrorCode.INCONSISTENT_DATA.getMessage, len, rowLen) + String.format(StorageErrorCode.INCONSISTENT_DATA.getMessage, len, rowLen) ) } rowCount = rowCount + 1 From 6fd06d3594d9504ce0ba258b6cc7037d13e6d27b Mon Sep 17 00:00:00 2001 From: peacewong Date: Fri, 19 May 2023 10:42:44 +0800 Subject: [PATCH 118/261] Fix build error --- .../linkis/storage/resultset/StorageResultSetReader.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/resultset/StorageResultSetReader.scala b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/resultset/StorageResultSetReader.scala index 9cd4bd2396..80db410746 100644 --- a/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/resultset/StorageResultSetReader.scala +++ b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/resultset/StorageResultSetReader.scala @@ -93,7 +93,7 @@ class StorageResultSetReader[K <: MetaData, V <: Record]( if (len != rowLen) { throw new StorageErrorException( StorageErrorCode.INCONSISTENT_DATA.getCode, - String.format(StorageErrorCode.INCONSISTENT_DATA.getMessage, len, rowLen) + String.format(StorageErrorCode.INCONSISTENT_DATA.getMessage, len.toString, rowLen.toString) ) } rowCount = rowCount + 1 From 7a744d1a067e36af77382021a41068ebb5713265 Mon Sep 17 00:00:00 2001 From: peacewong Date: Mon, 22 May 2023 22:08:46 +0800 Subject: [PATCH 119/261] Add creator level job limit --- .../linkis/scheduler/queue/Consumer.scala | 1 - .../linkis/scheduler/queue/GroupFactory.scala | 5 + .../queue/fifoqueue/FIFOUserConsumer.scala | 27 ++- .../conf/EntranceSpringConfiguration.java | 4 +- .../restful/EntranceConsumerRestfulApi.java | 88 ++++++++++ .../restful/EntranceMetricRestfulApi.java | 40 ++--- .../scheduler/CreatorECTypeDefaultConf.java | 91 ++++++++++ .../scheduler/EntranceFIFOUserConsumer.java | 69 ++++++++ .../EntranceParallelConsumerManager.java} | 19 ++- .../linkis/entrance/EntranceServer.scala | 11 +- .../GlobalConfigurationKeyValueCache.scala | 43 ----- .../entrance/conf/EntranceConfiguration.scala | 7 + ...tranceUserParallelOrchestratorPlugin.scala | 40 +---- .../scheduler/EntranceGroupFactory.scala | 153 ++++------------- .../scheduler/cache/ReadCacheConsumer.scala | 158 ------------------ .../cache/ReadCacheConsumerManager.scala | 44 ----- .../linkis/entrance/utils/EntranceUtils.scala | 108 ++++++++++++ .../label/client/InstanceLabelClient.scala | 25 ++- 18 files changed, 479 insertions(+), 454 deletions(-) create mode 100644 linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/restful/EntranceConsumerRestfulApi.java create mode 100644 linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/scheduler/CreatorECTypeDefaultConf.java create mode 100644 linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/scheduler/EntranceFIFOUserConsumer.java rename linkis-computation-governance/linkis-entrance/src/main/{scala/org/apache/linkis/entrance/scheduler/cache/CacheOutputExecuteResponse.scala => java/org/apache/linkis/entrance/scheduler/EntranceParallelConsumerManager.java} (52%) delete mode 100644 linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/cache/GlobalConfigurationKeyValueCache.scala delete mode 100644 linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/scheduler/cache/ReadCacheConsumer.scala delete mode 100644 linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/scheduler/cache/ReadCacheConsumerManager.scala create mode 100644 linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/utils/EntranceUtils.scala diff --git a/linkis-commons/linkis-scheduler/src/main/scala/org/apache/linkis/scheduler/queue/Consumer.scala b/linkis-commons/linkis-scheduler/src/main/scala/org/apache/linkis/scheduler/queue/Consumer.scala index 50dce2ca12..165a274362 100644 --- a/linkis-commons/linkis-scheduler/src/main/scala/org/apache/linkis/scheduler/queue/Consumer.scala +++ b/linkis-commons/linkis-scheduler/src/main/scala/org/apache/linkis/scheduler/queue/Consumer.scala @@ -41,7 +41,6 @@ abstract class Consumer(schedulerContext: SchedulerContext, executeService: Exec def start(): Unit def shutdown(): Unit = { - logger.info(s"$toString is ready to stop!") terminate = true logger.info(s"$toString stopped!") } diff --git a/linkis-commons/linkis-scheduler/src/main/scala/org/apache/linkis/scheduler/queue/GroupFactory.scala b/linkis-commons/linkis-scheduler/src/main/scala/org/apache/linkis/scheduler/queue/GroupFactory.scala index f3471b07dd..be1716f238 100644 --- a/linkis-commons/linkis-scheduler/src/main/scala/org/apache/linkis/scheduler/queue/GroupFactory.scala +++ b/linkis-commons/linkis-scheduler/src/main/scala/org/apache/linkis/scheduler/queue/GroupFactory.scala @@ -19,6 +19,11 @@ package org.apache.linkis.scheduler.queue abstract class GroupFactory { + /** + * Create a Group and set the concurrency limit of the group + * @param event + * @return + */ def getOrCreateGroup(event: SchedulerEvent): Group def getGroup(groupName: String): Group diff --git a/linkis-commons/linkis-scheduler/src/main/scala/org/apache/linkis/scheduler/queue/fifoqueue/FIFOUserConsumer.scala b/linkis-commons/linkis-scheduler/src/main/scala/org/apache/linkis/scheduler/queue/fifoqueue/FIFOUserConsumer.scala index 6ce6dc4a54..b330ccdbc9 100644 --- a/linkis-commons/linkis-scheduler/src/main/scala/org/apache/linkis/scheduler/queue/fifoqueue/FIFOUserConsumer.scala +++ b/linkis-commons/linkis-scheduler/src/main/scala/org/apache/linkis/scheduler/queue/fifoqueue/FIFOUserConsumer.scala @@ -73,6 +73,8 @@ class FIFOUserConsumer( override def getRunningEvents: Array[SchedulerEvent] = getEvents(e => e.isRunning || e.isWaitForRetry) + protected def getSchedulerContext: SchedulerContext = schedulerContext + private def getEvents(op: SchedulerEvent => Boolean): Array[SchedulerEvent] = { val result = ArrayBuffer[SchedulerEvent]() runningJobs.filter(_ != null).filter(x => op(x)).foreach(result += _) @@ -82,9 +84,9 @@ class FIFOUserConsumer( override def run(): Unit = { Thread.currentThread().setName(s"${toString}Thread") logger.info(s"$toString thread started!") - while (!terminate) { - Utils.tryAndError(loop()) - Utils.tryAndError(Thread.sleep(10)) + while (!terminate) Utils.tryAndError { + loop() + Thread.sleep(10) } logger.info(s"$toString thread stopped!") } @@ -92,14 +94,18 @@ class FIFOUserConsumer( protected def askExecutorGap(): Unit = {} /** - * Task scheduling interception is used to judge the rules of task operation, and to judge other task rules based on Group. For example, Entrance makes Creator-level task judgment. - * Note: Only when the interception condition is met can it return, otherwise the rule will be invalid + * Task scheduling interception is used to judge the rules of task operation, and to judge other + * task rules based on Group. For example, Entrance makes Creator-level task judgment. */ - protected def runScheduleIntercept(): Unit = { - + protected def runScheduleIntercept(): Boolean = { + true } protected def loop(): Unit = { + if (!runScheduleIntercept()) { + Utils.tryQuietly(Thread.sleep(1000)) + return + } var isRetryJob = false def getWaitForRetryEvent: Option[SchedulerEvent] = { val waitForRetryJobs = runningJobs.filter(job => job != null && job.isJobCanRetry) @@ -127,7 +133,12 @@ class FIFOUserConsumer( if ( takeEvent.exists(e => Utils.tryCatch(e.turnToScheduled()) { t => - takeEvent.get.asInstanceOf[Job].onFailure("Failed to change the job status to Scheduled(Job状态翻转为Scheduled失败)", t) + takeEvent.get + .asInstanceOf[Job] + .onFailure( + "Failed to change the job status to Scheduled(Job状态翻转为Scheduled失败)", + t + ) false } ) diff --git a/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/conf/EntranceSpringConfiguration.java b/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/conf/EntranceSpringConfiguration.java index 0bf27a68b3..cf520c3823 100644 --- a/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/conf/EntranceSpringConfiguration.java +++ b/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/conf/EntranceSpringConfiguration.java @@ -42,6 +42,7 @@ import org.apache.linkis.entrance.persistence.QueryPersistenceManager; import org.apache.linkis.entrance.persistence.ResultSetEngine; import org.apache.linkis.entrance.scheduler.EntranceGroupFactory; +import org.apache.linkis.entrance.scheduler.EntranceParallelConsumerManager; import org.apache.linkis.entrance.scheduler.EntranceSchedulerContext; import org.apache.linkis.orchestrator.ecm.EngineConnManagerBuilder; import org.apache.linkis.orchestrator.ecm.EngineConnManagerBuilder$; @@ -51,7 +52,6 @@ import org.apache.linkis.scheduler.executer.ExecutorManager; import org.apache.linkis.scheduler.queue.ConsumerManager; import org.apache.linkis.scheduler.queue.GroupFactory; -import org.apache.linkis.scheduler.queue.parallelqueue.ParallelConsumerManager; import org.apache.linkis.scheduler.queue.parallelqueue.ParallelScheduler; import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; @@ -190,7 +190,7 @@ public GroupFactory groupFactory() { @Bean @ConditionalOnMissingBean public ConsumerManager consumerManager() { - return new ParallelConsumerManager( + return new EntranceParallelConsumerManager( ENTRANCE_SCHEDULER_MAX_PARALLELISM_USERS().getValue(), "EntranceJobScheduler"); } diff --git a/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/restful/EntranceConsumerRestfulApi.java b/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/restful/EntranceConsumerRestfulApi.java new file mode 100644 index 0000000000..424e7ca170 --- /dev/null +++ b/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/restful/EntranceConsumerRestfulApi.java @@ -0,0 +1,88 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.entrance.restful; + +import org.apache.linkis.common.conf.Configuration; +import org.apache.linkis.entrance.EntranceServer; +import org.apache.linkis.scheduler.queue.ConsumerManager; +import org.apache.linkis.server.Message; +import org.apache.linkis.server.utils.ModuleUserUtils; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestMethod; +import org.springframework.web.bind.annotation.RequestParam; +import org.springframework.web.bind.annotation.RestController; + +import javax.servlet.http.HttpServletRequest; + +import io.swagger.annotations.Api; +import io.swagger.annotations.ApiOperation; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +@Api(tags = "entrance lable manager") +@RestController +@RequestMapping(path = "/entrance/operation/consumer") +public class EntranceConsumerRestfulApi { + + private EntranceServer entranceServer; + + private static final Logger logger = LoggerFactory.getLogger(EntranceConsumerRestfulApi.class); + + @Autowired + public void setEntranceServer(EntranceServer entranceServer) { + this.entranceServer = entranceServer; + } + + @ApiOperation(value = "kill-consumer", notes = "kill consumer", response = Message.class) + @RequestMapping(path = "/kill", method = RequestMethod.GET) + public Message killConsumer( + HttpServletRequest req, @RequestParam(value = "groupName") String groupName) { + String operationUser = ModuleUserUtils.getOperationUser(req, "kill consumer"); + if (Configuration.isNotAdmin(operationUser)) { + return Message.error("only admin can do this"); + } + logger.info("user {} to kill consumer {}", operationUser, groupName); + ConsumerManager consumerManager = + entranceServer + .getEntranceContext() + .getOrCreateScheduler() + .getSchedulerContext() + .getOrCreateConsumerManager(); + consumerManager.destroyConsumer(groupName); + logger.info("user {} finished to kill consumer {}", operationUser, groupName); + return Message.ok(); + } + + @ApiOperation(value = "consumer-info", notes = "list consumers info", response = Message.class) + @RequestMapping(path = "/info", method = RequestMethod.GET) + public Message countConsumer(HttpServletRequest req) { + String operationUser = ModuleUserUtils.getOperationUser(req, "kill consumer"); + if (Configuration.isNotAdmin(operationUser)) { + return Message.error("only admin can do this"); + } + ConsumerManager consumerManager = + entranceServer + .getEntranceContext() + .getOrCreateScheduler() + .getSchedulerContext() + .getOrCreateConsumerManager(); + return Message.ok().data("consumerNum", consumerManager.listConsumers().length); + } +} diff --git a/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/restful/EntranceMetricRestfulApi.java b/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/restful/EntranceMetricRestfulApi.java index 7d36df8fec..7b487352d5 100644 --- a/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/restful/EntranceMetricRestfulApi.java +++ b/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/restful/EntranceMetricRestfulApi.java @@ -20,8 +20,7 @@ import org.apache.linkis.common.conf.Configuration; import org.apache.linkis.entrance.EntranceServer; import org.apache.linkis.entrance.execute.EntranceJob; -import org.apache.linkis.manager.label.entity.engine.EngineTypeLabel; -import org.apache.linkis.manager.label.utils.LabelUtil; +import org.apache.linkis.entrance.scheduler.CreatorECTypeDefaultConf; import org.apache.linkis.server.Message; import org.apache.linkis.server.utils.ModuleUserUtils; @@ -67,7 +66,7 @@ public Message taskinfo( HttpServletRequest req, @RequestParam(value = "user", required = false) String user, @RequestParam(value = "creator", required = false) String creator, - @RequestParam(value = "engineTypeLabel", required = false) String engineTypeLabelValue) { + @RequestParam(value = "ecType", required = false) String ecType) { String userName = ModuleUserUtils.getOperationUser(req, "taskinfo"); String queryUser = user; if (Configuration.isNotAdmin(userName)) { @@ -83,23 +82,12 @@ public Message taskinfo( } else if (StringUtils.isBlank(creator)) { filterWords = queryUser; } - EntranceJob[] undoneTasks = entranceServer.getAllUndoneTask(filterWords); - int taskNumber = 0; + EntranceJob[] undoneTasks = entranceServer.getAllUndoneTask(filterWords, ecType); int runningNumber = 0; int queuedNumber = 0; if (null != undoneTasks) { for (EntranceJob task : undoneTasks) { - if (StringUtils.isNotBlank(engineTypeLabelValue)) { - EngineTypeLabel engineTypeLabel = - LabelUtil.getEngineTypeLabel(task.getJobRequest().getLabels()); - // Task types do not match, do not count - if (null == engineTypeLabel - || !engineTypeLabelValue.equalsIgnoreCase(engineTypeLabel.getStringValue())) { - continue; - } - } - taskNumber++; if (task.isRunning()) { runningNumber++; } else { @@ -107,17 +95,25 @@ public Message taskinfo( } } } - return Message.ok("success") - .data("taskNumber", taskNumber) - .data("runningNumber", runningNumber) - .data("queuedNumber", queuedNumber); + Message resp = + Message.ok("success") + .data("taskNumber", undoneTasks.length) + .data("runningNumber", runningNumber) + .data("queuedNumber", queuedNumber); + if (StringUtils.isNoneBlank(creator, ecType)) { + int creatorECTypeMaxRunningJobs = + CreatorECTypeDefaultConf.getCreatorECTypeMaxRunningJobs(creator, ecType); + resp.data("creatorECTypeMaxRunningJobs", creatorECTypeMaxRunningJobs); + resp.data("limitExceeded", runningNumber > creatorECTypeMaxRunningJobs); + } + return resp; } - @ApiOperation(value = "Status", notes = "get running task number ", response = Message.class) + @ApiOperation(value = "runningtask", notes = "get running task number ", response = Message.class) @RequestMapping(path = "/runningtask", method = RequestMethod.GET) - public Message status(HttpServletRequest req) { + public Message runningtask(HttpServletRequest req) { ModuleUserUtils.getOperationUser(req, "runningtask"); - EntranceJob[] undoneTasks = entranceServer.getAllUndoneTask(""); + EntranceJob[] undoneTasks = entranceServer.getAllUndoneTask("", null); Boolean isCompleted = false; if (null == undoneTasks || undoneTasks.length < 1) { isCompleted = true; diff --git a/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/scheduler/CreatorECTypeDefaultConf.java b/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/scheduler/CreatorECTypeDefaultConf.java new file mode 100644 index 0000000000..8a28a5e92f --- /dev/null +++ b/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/scheduler/CreatorECTypeDefaultConf.java @@ -0,0 +1,91 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.entrance.scheduler; + +import org.apache.linkis.common.conf.Configuration; +import org.apache.linkis.entrance.conf.EntranceConfiguration; +import org.apache.linkis.entrance.utils.EntranceUtils; +import org.apache.linkis.governance.common.protocol.conf.RequestQueryEngineConfig; +import org.apache.linkis.governance.common.protocol.conf.ResponseQueryConfig; +import org.apache.linkis.manager.label.entity.engine.EngineTypeLabel; +import org.apache.linkis.manager.label.entity.engine.UserCreatorLabel; +import org.apache.linkis.rpc.Sender; + +import org.apache.commons.lang3.StringUtils; + +import java.util.concurrent.TimeUnit; + +import scala.Tuple2; + +import com.google.common.cache.CacheBuilder; +import com.google.common.cache.CacheLoader; +import com.google.common.cache.LoadingCache; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class CreatorECTypeDefaultConf { + + private static final Logger logger = LoggerFactory.getLogger(CreatorECTypeDefaultConf.class); + + public static Sender confSender = + Sender.getSender( + Configuration.CLOUD_CONSOLE_CONFIGURATION_SPRING_APPLICATION_NAME().getValue()); + + private static LoadingCache confCache = + CacheBuilder.newBuilder() + .maximumSize(1000) + .expireAfterWrite(1, TimeUnit.HOURS) + .build( + new CacheLoader() { + @Override + public Integer load(String key) throws Exception { + Tuple2 tuple2 = + EntranceUtils.fromKeyGetLabels(key); + RequestQueryEngineConfig requestQueryEngineConfig = + new RequestQueryEngineConfig(tuple2._1, tuple2._2(), null); + int jobLimit = + (int) EntranceConfiguration.ENTRANCE_CREATOR_JOB_LIMIT().getValue(); + try { + Object response = confSender.ask(requestQueryEngineConfig); + if (response instanceof ResponseQueryConfig) { + jobLimit = + (int) + EntranceConfiguration.ENTRANCE_CREATOR_JOB_LIMIT() + .getValue(((ResponseQueryConfig) response).getKeyAndValue()); + } + } catch (Exception e) { + logger.warn("Failed to get key {} from conf", key, e); + } + return jobLimit; + } + }); + + public static int getCreatorECTypeMaxRunningJobs(String creator, String ecType) { + int jobLimit = (int) EntranceConfiguration.ENTRANCE_CREATOR_JOB_LIMIT().getValue(); + if (StringUtils.isNoneBlank(creator, ecType)) { + try { + String key = EntranceUtils.getDefaultCreatorECTypeKey(creator, ecType); + jobLimit = confCache.get(key); + } catch (Exception e) { + logger.warn("Failed to get key creator {} ecType {} from cache", creator, ecType, e); + } + } + int entranceNumber = EntranceUtils.getRunningEntranceNumber(); + return jobLimit / entranceNumber; + } +} diff --git a/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/scheduler/EntranceFIFOUserConsumer.java b/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/scheduler/EntranceFIFOUserConsumer.java new file mode 100644 index 0000000000..387606256a --- /dev/null +++ b/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/scheduler/EntranceFIFOUserConsumer.java @@ -0,0 +1,69 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.entrance.scheduler; + +import org.apache.linkis.scheduler.SchedulerContext; +import org.apache.linkis.scheduler.queue.Consumer; +import org.apache.linkis.scheduler.queue.Group; +import org.apache.linkis.scheduler.queue.fifoqueue.FIFOUserConsumer; + +import java.util.concurrent.ExecutorService; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class EntranceFIFOUserConsumer extends FIFOUserConsumer { + + private static final Logger logger = LoggerFactory.getLogger(EntranceFIFOUserConsumer.class); + + public EntranceFIFOUserConsumer( + SchedulerContext schedulerContext, ExecutorService executeService, Group group) { + super(schedulerContext, executeService, group); + } + + @Override + public boolean runScheduleIntercept() { + Consumer[] consumers = getSchedulerContext().getOrCreateConsumerManager().listConsumers(); + int creatorRunningJobNum = 0; + String[] groupNames = getGroup().getGroupName().split("_"); + if (groupNames.length < 3) { + return true; + } + String creatorName = groupNames[0]; + String ecType = groupNames[2]; + for (Consumer consumer : consumers) { + String groupName = consumer.getGroup().getGroupName(); + if (groupName.startsWith(creatorName) && groupName.endsWith(ecType)) { + creatorRunningJobNum += consumer.getRunningEvents().length; + } + } + int creatorECTypeMaxRunningJobs = + CreatorECTypeDefaultConf.getCreatorECTypeMaxRunningJobs(creatorName, ecType); + if (creatorRunningJobNum > creatorECTypeMaxRunningJobs) { + logger.error( + "Creator: {} EC: {} there are currently {} jobs running that exceed the maximum limit: {}", + creatorName, + ecType, + creatorRunningJobNum, + creatorECTypeMaxRunningJobs); + return false; + } else { + return true; + } + } +} diff --git a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/scheduler/cache/CacheOutputExecuteResponse.scala b/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/scheduler/EntranceParallelConsumerManager.java similarity index 52% rename from linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/scheduler/cache/CacheOutputExecuteResponse.scala rename to linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/scheduler/EntranceParallelConsumerManager.java index 47a6ce9e9e..98f0929ee9 100644 --- a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/scheduler/cache/CacheOutputExecuteResponse.scala +++ b/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/scheduler/EntranceParallelConsumerManager.java @@ -15,10 +15,21 @@ * limitations under the License. */ -package org.apache.linkis.entrance.scheduler.cache +package org.apache.linkis.entrance.scheduler; -import org.apache.linkis.scheduler.executer.OutputExecuteResponse +import org.apache.linkis.scheduler.queue.Group; +import org.apache.linkis.scheduler.queue.fifoqueue.FIFOUserConsumer; +import org.apache.linkis.scheduler.queue.parallelqueue.ParallelConsumerManager; -case class CacheOutputExecuteResponse(alias: String, output: String) extends OutputExecuteResponse { - override def getOutput: String = output +public class EntranceParallelConsumerManager extends ParallelConsumerManager { + + public EntranceParallelConsumerManager(int maxParallelismUsers, String schedulerName) { + super(maxParallelismUsers, schedulerName); + } + + @Override + public FIFOUserConsumer createConsumer(String groupName) { + Group group = getSchedulerContext().getOrCreateGroupFactory().getGroup(groupName); + return new EntranceFIFOUserConsumer(getSchedulerContext(), getOrCreateExecutorService(), group); + } } diff --git a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/EntranceServer.scala b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/EntranceServer.scala index c185b78098..9793e39506 100644 --- a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/EntranceServer.scala +++ b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/EntranceServer.scala @@ -224,7 +224,7 @@ abstract class EntranceServer extends Logging { entranceWebSocketService } else None - def getAllUndoneTask(filterWords: String): Array[EntranceJob] = { + def getAllUndoneTask(filterWords: String, ecType: String = null): Array[EntranceJob] = { val consumers = getEntranceContext .getOrCreateScheduler() .getSchedulerContext @@ -232,7 +232,14 @@ abstract class EntranceServer extends Logging { .listConsumers() .toSet val filterConsumer = if (StringUtils.isNotBlank(filterWords)) { - consumers.filter(_.getGroup.getGroupName.contains(filterWords)) + if (StringUtils.isNotBlank(ecType)) { + consumers.filter(consumer => + consumer.getGroup.getGroupName.contains(filterWords) && consumer.getGroup.getGroupName + .contains(ecType) + ) + } else { + consumers.filter(_.getGroup.getGroupName.contains(filterWords)) + } } else { consumers } diff --git a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/cache/GlobalConfigurationKeyValueCache.scala b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/cache/GlobalConfigurationKeyValueCache.scala deleted file mode 100644 index d4190d6ca4..0000000000 --- a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/cache/GlobalConfigurationKeyValueCache.scala +++ /dev/null @@ -1,43 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.entrance.cache - -import org.apache.linkis.common.conf.Configuration -import org.apache.linkis.governance.common.entity.job.JobRequest -import org.apache.linkis.governance.common.protocol.conf.{ - RequestQueryGlobalConfig, - ResponseQueryConfig -} -import org.apache.linkis.protocol.CacheableProtocol -import org.apache.linkis.rpc.RPCMapCache - -import java.util - -object GlobalConfigurationKeyValueCache - extends RPCMapCache[JobRequest, String, String]( - Configuration.CLOUD_CONSOLE_CONFIGURATION_SPRING_APPLICATION_NAME.getValue - ) { - - override protected def createRequest(jobReq: JobRequest): CacheableProtocol = - RequestQueryGlobalConfig(jobReq.getExecuteUser) - - override protected def createMap(any: Any): util.Map[String, String] = any match { - case response: ResponseQueryConfig => response.getKeyAndValue - } - -} diff --git a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/conf/EntranceConfiguration.scala b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/conf/EntranceConfiguration.scala index 5c61ce0b3b..b7380896f8 100644 --- a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/conf/EntranceConfiguration.scala +++ b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/conf/EntranceConfiguration.scala @@ -223,4 +223,11 @@ object EntranceConfiguration { val ENABLE_ENTRANCE_DIRTY_DATA_CLEAR = CommonVars("linkis.entrance.auto.clean.dirty.data.enable", true) + val ENTRANCE_CREATOR_JOB_LIMIT: CommonVars[Int] = + CommonVars[Int]( + "linkis.entrance.creator.job.concurrency.limit", + 10000, + "Creator task concurrency limit parameters" + ) + } diff --git a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/orchestrator/plugin/EntranceUserParallelOrchestratorPlugin.scala b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/orchestrator/plugin/EntranceUserParallelOrchestratorPlugin.scala index 4b9b4570f1..e5c657023e 100644 --- a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/orchestrator/plugin/EntranceUserParallelOrchestratorPlugin.scala +++ b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/orchestrator/plugin/EntranceUserParallelOrchestratorPlugin.scala @@ -20,19 +20,15 @@ package org.apache.linkis.entrance.orchestrator.plugin import org.apache.linkis.common.conf.Configuration import org.apache.linkis.common.utils.{Logging, Utils} import org.apache.linkis.entrance.conf.EntranceConfiguration +import org.apache.linkis.entrance.utils.EntranceUtils import org.apache.linkis.governance.common.protocol.conf.{ RequestQueryEngineConfigWithGlobalConfig, ResponseQueryConfig } -import org.apache.linkis.manager.label.builder.factory.LabelBuilderFactoryContext -import org.apache.linkis.manager.label.constant.LabelKeyConstant import org.apache.linkis.manager.label.entity.Label import org.apache.linkis.manager.label.entity.engine.{EngineTypeLabel, UserCreatorLabel} import org.apache.linkis.orchestrator.plugin.UserParallelOrchestratorPlugin import org.apache.linkis.rpc.Sender -import org.apache.linkis.server.BDPJettyServerHelper - -import org.apache.commons.lang3.StringUtils import java.util import java.util.concurrent.TimeUnit @@ -43,10 +39,6 @@ import com.google.common.cache.{CacheBuilder, CacheLoader, LoadingCache} class EntranceUserParallelOrchestratorPlugin extends UserParallelOrchestratorPlugin with Logging { - private val SPLIT = "," - - private val labelFactory = LabelBuilderFactoryContext.getLabelBuilderFactory - private def getDefaultMaxRuningNum: Int = { EntranceConfiguration.WDS_LINKIS_INSTANCE.getHotValue() } @@ -62,7 +54,7 @@ class EntranceUserParallelOrchestratorPlugin extends UserParallelOrchestratorPlu .build(new CacheLoader[String, Integer]() { override def load(key: String): Integer = { - val (userCreatorLabel, engineTypeLabel) = fromKeyGetLabels(key) + val (userCreatorLabel, engineTypeLabel) = EntranceUtils.fromKeyGetLabels(key) val keyAndValue = Utils.tryAndWarnMsg { sender .ask(RequestQueryEngineConfigWithGlobalConfig(userCreatorLabel, engineTypeLabel)) @@ -75,10 +67,8 @@ class EntranceUserParallelOrchestratorPlugin extends UserParallelOrchestratorPlu null == keyAndValue || !keyAndValue .containsKey(EntranceConfiguration.WDS_LINKIS_INSTANCE.key) ) { - logger.error( - s"cannot found user configuration key:${EntranceConfiguration.WDS_LINKIS_INSTANCE.key}," + - s"will use default value ${EntranceConfiguration.WDS_LINKIS_INSTANCE.getHotValue()}。All config map: ${BDPJettyServerHelper.gson - .toJson(keyAndValue)}" + logger.warn( + s"cannot found user configuration key:${EntranceConfiguration.WDS_LINKIS_INSTANCE.key}," + s"will use default value " ) } val maxRunningJobs = EntranceConfiguration.WDS_LINKIS_INSTANCE.getValue(keyAndValue, true) @@ -102,27 +92,7 @@ class EntranceUserParallelOrchestratorPlugin extends UserParallelOrchestratorPlu if (null == userCreatorLabel || null == engineTypeLabel) { return getDefaultMaxRuningNum } - configCache.get(getKey(userCreatorLabel, engineTypeLabel)) - } - - private def getKey( - userCreatorLabel: UserCreatorLabel, - engineTypeLabel: EngineTypeLabel - ): String = { - userCreatorLabel.getStringValue + SPLIT + engineTypeLabel.getStringValue - } - - private def fromKeyGetLabels(key: String): (UserCreatorLabel, EngineTypeLabel) = { - if (StringUtils.isBlank(key)) (null, null) - else { - val labelStringValues = key.split(SPLIT) - if (labelStringValues.length < 2) return (null, null) - val userCreatorLabel = labelFactory - .createLabel[UserCreatorLabel](LabelKeyConstant.USER_CREATOR_TYPE_KEY, labelStringValues(0)) - val engineTypeLabel = labelFactory - .createLabel[EngineTypeLabel](LabelKeyConstant.ENGINE_TYPE_KEY, labelStringValues(1)) - (userCreatorLabel, engineTypeLabel) - } + configCache.get(EntranceUtils.getUserCreatorEcTypeKey(userCreatorLabel, engineTypeLabel)) } override def isReady: Boolean = true diff --git a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/scheduler/EntranceGroupFactory.scala b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/scheduler/EntranceGroupFactory.scala index 7f16dd2463..a436178880 100644 --- a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/scheduler/EntranceGroupFactory.scala +++ b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/scheduler/EntranceGroupFactory.scala @@ -17,30 +17,20 @@ package org.apache.linkis.entrance.scheduler -import org.apache.linkis.common.ServiceInstance import org.apache.linkis.common.conf.{CommonVars, Configuration} import org.apache.linkis.common.utils.{Logging, Utils} import org.apache.linkis.entrance.conf.EntranceConfiguration import org.apache.linkis.entrance.errorcode.EntranceErrorCodeSummary._ import org.apache.linkis.entrance.exception.{EntranceErrorCode, EntranceErrorException} import org.apache.linkis.entrance.execute.EntranceJob +import org.apache.linkis.entrance.utils.EntranceUtils import org.apache.linkis.governance.common.protocol.conf.{ RequestQueryEngineConfigWithGlobalConfig, ResponseQueryConfig } -import org.apache.linkis.instance.label.client.InstanceLabelClient -import org.apache.linkis.manager.label.builder.factory.LabelBuilderFactoryContext -import org.apache.linkis.manager.label.constant.{LabelKeyConstant, LabelValueConstant} import org.apache.linkis.manager.label.entity.Label -import org.apache.linkis.manager.label.entity.engine.{ - ConcurrentEngineConnLabel, - EngineTypeLabel, - UserCreatorLabel -} -import org.apache.linkis.manager.label.entity.route.RouteLabel +import org.apache.linkis.manager.label.entity.engine.{EngineTypeLabel, UserCreatorLabel} import org.apache.linkis.manager.label.utils.LabelUtil -import org.apache.linkis.protocol.constants.TaskConstant -import org.apache.linkis.protocol.utils.TaskUtils import org.apache.linkis.rpc.Sender import org.apache.linkis.scheduler.queue.{Group, GroupFactory, SchedulerEvent} import org.apache.linkis.scheduler.queue.parallelqueue.ParallelGroup @@ -51,8 +41,6 @@ import java.util import java.util.concurrent.TimeUnit import java.util.regex.Pattern -import scala.collection.JavaConverters._ - import com.google.common.cache.{Cache, CacheBuilder} class EntranceGroupFactory extends GroupFactory with Logging { @@ -81,29 +69,16 @@ class EntranceGroupFactory extends GroupFactory with Logging { } override def getOrCreateGroup(event: SchedulerEvent): Group = { - val (labels, params) = event match { + val labels = event match { case job: EntranceJob => - (job.getJobRequest.getLabels, job.getJobRequest.getParams) + job.getJobRequest.getLabels + case _ => + throw new EntranceErrorException(LABEL_NOT_NULL.getErrorCode, LABEL_NOT_NULL.getErrorDesc) } - val groupName = EntranceGroupFactory.getGroupNameByLabels(labels, params) + val groupName = EntranceGroupFactory.getGroupNameByLabels(labels) val cacheGroup = groupNameToGroups.getIfPresent(groupName) if (null == cacheGroup) synchronized { val maxAskExecutorTimes = EntranceConfiguration.MAX_ASK_EXECUTOR_TIME.getValue.toLong - if (groupName.startsWith(EntranceGroupFactory.CONCURRENT)) { - if (null == groupNameToGroups.getIfPresent(groupName)) synchronized { - if (null == groupNameToGroups.getIfPresent(groupName)) { - val group = new ParallelGroup( - groupName, - 100, - EntranceConfiguration.CONCURRENT_FACTORY_MAX_CAPACITY.getValue - ) - group.setMaxRunningJobs(EntranceConfiguration.CONCURRENT_MAX_RUNNING_JOBS.getValue) - group.setMaxAskExecutorTimes(EntranceConfiguration.CONCURRENT_EXECUTOR_TIME.getValue) - groupNameToGroups.put(groupName, group) - return group - } - } - } val sender: Sender = Sender.getSender(Configuration.CLOUD_CONSOLE_CONFIGURATION_SPRING_APPLICATION_NAME.getValue) val userCreatorLabel: UserCreatorLabel = LabelUtil.getUserCreatorLabel(labels) @@ -141,8 +116,11 @@ class EntranceGroupFactory extends GroupFactory with Logging { group.setMaxRunningJobs(maxRunningJobs) group.setMaxAskExecutorTimes(maxAskExecutorTimes) groupNameToGroups.put(groupName, group) + group + } + else { + cacheGroup } - groupNameToGroups.getIfPresent(groupName) } override def getGroup(groupName: String): Group = { @@ -156,105 +134,40 @@ class EntranceGroupFactory extends GroupFactory with Logging { group } + /** + * User task concurrency control is controlled for multiple Entrances, which will be evenly + * distributed based on the number of existing Entrances + * @param keyAndValue + * @return + */ private def getUserMaxRunningJobs(keyAndValue: util.Map[String, String]): Int = { - var userDefinedRunningJobs = EntranceConfiguration.WDS_LINKIS_INSTANCE.getValue(keyAndValue) - var entranceNum = Sender.getInstances(Sender.getThisServiceInstance.getApplicationName).length - val labelList = new util.ArrayList[Label[_]]() - val offlineRouteLabel = LabelBuilderFactoryContext.getLabelBuilderFactory - .createLabel[RouteLabel](LabelKeyConstant.ROUTE_KEY, LabelValueConstant.OFFLINE_VALUE) - labelList.add(offlineRouteLabel) - var offlineIns: Array[ServiceInstance] = null - Utils.tryAndWarn { - offlineIns = InstanceLabelClient.getInstance - .getInstanceFromLabel(labelList) - .asScala - .filter(l => - null != l && l.getApplicationName - .equalsIgnoreCase(Sender.getThisServiceInstance.getApplicationName) - ) - .toArray - } - if (null != offlineIns) { - logger.info(s"There are ${offlineIns.length} offlining instance.") - entranceNum = entranceNum - offlineIns.length - } - /* - Sender.getInstances may get 0 instances due to cache in Sender. So this instance is the one instance. - */ - if (0 >= entranceNum) { - logger.error( - s"Got ${entranceNum} ${Sender.getThisServiceInstance.getApplicationName} instances." - ) - entranceNum = 1 - } + val userDefinedRunningJobs = EntranceConfiguration.WDS_LINKIS_INSTANCE.getValue(keyAndValue) + val entranceNum = EntranceUtils.getRunningEntranceNumber() Math.max( EntranceConfiguration.ENTRANCE_INSTANCE_MIN.getValue, userDefinedRunningJobs / entranceNum - ); + ) } } object EntranceGroupFactory { - val CACHE = "_Cache" - - val CONCURRENT = "Concurrent_" - - def getGroupName( - creator: String, - user: String, - params: util.Map[String, AnyRef] = new util.HashMap[String, AnyRef] - ): String = { - val runtime = TaskUtils.getRuntimeMap(params) - val cache = - if ( - runtime.get(TaskConstant.READ_FROM_CACHE) != null && runtime - .get(TaskConstant.READ_FROM_CACHE) - .asInstanceOf[Boolean] - ) { - CACHE - } else "" - if (StringUtils.isNotEmpty(creator)) creator + "_" + user + cache - else EntranceConfiguration.DEFAULT_REQUEST_APPLICATION_NAME.getValue + "_" + user + cache - } - - def getGroupNameByLabels( - labels: java.util.List[Label[_]], - params: util.Map[String, AnyRef] = new util.HashMap[String, AnyRef] - ): String = { - - val userCreator = labels.asScala.find(_.isInstanceOf[UserCreatorLabel]) - val engineType = labels.asScala.find(_.isInstanceOf[EngineTypeLabel]) - val concurrent = labels.asScala.find(_.isInstanceOf[ConcurrentEngineConnLabel]) - if (userCreator.isEmpty || engineType.isEmpty) { + /** + * Entrance group rule creator_username_engineType eg:IDE_PEACEWONG_SPARK + * @param labels + * @param params + * @return + */ + def getGroupNameByLabels(labels: java.util.List[Label[_]]): String = { + val userCreatorLabel = LabelUtil.getUserCreatorLabel(labels) + val engineTypeLabel = LabelUtil.getEngineTypeLabel(labels) + if (null == userCreatorLabel || null == engineTypeLabel) { throw new EntranceErrorException(LABEL_NOT_NULL.getErrorCode, LABEL_NOT_NULL.getErrorDesc) } - - if (concurrent.isDefined) { - - val engineTypeLabel = engineType.get.asInstanceOf[EngineTypeLabel] - val groupName = CONCURRENT + engineTypeLabel.getEngineType - groupName - - } else { - val userCreatorLabel = userCreator.get.asInstanceOf[UserCreatorLabel] - - val engineTypeLabel = engineType.get.asInstanceOf[EngineTypeLabel] - - val runtime = TaskUtils.getRuntimeMap(params) - val cache = - if ( - runtime.get(TaskConstant.READ_FROM_CACHE) != null && runtime - .get(TaskConstant.READ_FROM_CACHE) - .asInstanceOf[Boolean] - ) { - CACHE - } else "" - val groupName = - userCreatorLabel.getCreator + "_" + userCreatorLabel.getUser + "_" + engineTypeLabel.getEngineType + cache - groupName - } + val groupName = + userCreatorLabel.getCreator + "_" + userCreatorLabel.getUser + "_" + engineTypeLabel.getEngineType + groupName } } diff --git a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/scheduler/cache/ReadCacheConsumer.scala b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/scheduler/cache/ReadCacheConsumer.scala deleted file mode 100644 index 65bbbd39b4..0000000000 --- a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/scheduler/cache/ReadCacheConsumer.scala +++ /dev/null @@ -1,158 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.entrance.scheduler.cache - -import org.apache.linkis.common.io.FsPath -import org.apache.linkis.common.utils.Utils -import org.apache.linkis.entrance.errorcode.EntranceErrorCodeSummary._ -import org.apache.linkis.entrance.exception.CacheNotReadyException -import org.apache.linkis.entrance.execute.EntranceJob -import org.apache.linkis.entrance.persistence.PersistenceManager -import org.apache.linkis.entrance.utils.JobHistoryHelper -import org.apache.linkis.governance.common.entity.job.JobRequest -import org.apache.linkis.manager.label.constant.LabelKeyConstant -import org.apache.linkis.protocol.constants.TaskConstant -import org.apache.linkis.protocol.utils.TaskUtils -import org.apache.linkis.scheduler.SchedulerContext -import org.apache.linkis.scheduler.errorcode.LinkisSchedulerErrorCodeSummary._ -import org.apache.linkis.scheduler.exception.SchedulerErrorException -import org.apache.linkis.scheduler.executer.SuccessExecuteResponse -import org.apache.linkis.scheduler.queue.Group -import org.apache.linkis.scheduler.queue.fifoqueue.FIFOUserConsumer -import org.apache.linkis.server.BDPJettyServerHelper -import org.apache.linkis.storage.FSFactory -import org.apache.linkis.storage.fs.FileSystem - -import org.apache.commons.io.FilenameUtils -import org.apache.commons.lang3.StringUtils - -import java.util.concurrent.ExecutorService - -import scala.collection.JavaConverters._ - -import com.google.common.collect.Lists - -class ReadCacheConsumer( - schedulerContext: SchedulerContext, - executeService: ExecutorService, - private var group: Group, - persistenceManager: PersistenceManager -) extends FIFOUserConsumer(schedulerContext, executeService, group) { - - override protected def loop(): Unit = { - val event = Option(getConsumeQueue.take()) - event.foreach { - case job: EntranceJob => - job.getJobRequest match { - case jobRequest: JobRequest => - Utils.tryCatch { - val engineTpyeLabel = jobRequest.getLabels.asScala - .filter(l => l.getLabelKey.equalsIgnoreCase(LabelKeyConstant.ENGINE_TYPE_KEY)) - .headOption - .getOrElse(null) - val labelStrList = jobRequest.getLabels.asScala.map { case l => - l.getStringValue - }.toList - if (null == engineTpyeLabel) { - logger.error( - "Invalid engineType null, cannot process. jobReq : " + BDPJettyServerHelper.gson - .toJson(jobRequest) - ) - throw CacheNotReadyException( - INVALID_ENGINETYPE_NULL.getErrorCode, - INVALID_ENGINETYPE_NULL.getErrorDesc - ) - } - val readCacheBefore: Long = TaskUtils - .getRuntimeMap(job.getParams) - .getOrDefault(TaskConstant.READ_CACHE_BEFORE, 300L: java.lang.Long) - .asInstanceOf[Long] - val cacheResult = JobHistoryHelper.getCache( - jobRequest.getExecutionCode, - jobRequest.getExecuteUser, - labelStrList.asJava, - readCacheBefore - ) - if (cacheResult != null && StringUtils.isNotBlank(cacheResult.getResultLocation)) { - val resultSets = listResults(cacheResult.getResultLocation, job.getUser) - if (resultSets.size() > 0) { - for (resultSet: FsPath <- resultSets.asScala) { - val alias = FilenameUtils.getBaseName(resultSet.getPath) - val output = FsPath - .getFsPath( - cacheResult.getResultLocation, - FilenameUtils.getName(resultSet.getPath) - ) - .getSchemaPath -// persistenceManager.onResultSetCreated(job, new CacheOutputExecuteResponse(alias, output)) - throw CacheNotReadyException( - INVALID_RESULTSETS.getErrorCode, - INVALID_RESULTSETS.getErrorDesc - ) - // todo check - } -// persistenceManager.onResultSizeCreated(job, resultSets.size()) - } - val runtime = TaskUtils.getRuntimeMap(job.getParams) - runtime.put(TaskConstant.CACHE, java.lang.Boolean.FALSE) - TaskUtils.addRuntimeMap(job.getParams, runtime) - job.transitionCompleted(SuccessExecuteResponse(), "Result found in cache") - } else { - logger.info("Cache not found, submit to normal consumer.") - submitToExecute(job) - } - } { t => - logger.warn("Read cache failed, submit to normal consumer: ", t) - submitToExecute(job) - } - case _ => - } - case _ => - } - } - - private def listResults(resultLocation: String, user: String) = { - val dirPath = FsPath.getFsPath(resultLocation) - val fileSystem = FSFactory.getFsByProxyUser(dirPath, user).asInstanceOf[FileSystem] - Utils.tryFinally { - fileSystem.init(null) - if (fileSystem.exists(dirPath)) { - fileSystem.listPathWithError(dirPath).getFsPaths - } else { - Lists.newArrayList[FsPath]() - } - }(Utils.tryQuietly(fileSystem.close())) - } - - private def submitToExecute(job: EntranceJob): Unit = { - val runtime = TaskUtils.getRuntimeMap(job.getParams) - runtime.put(TaskConstant.READ_FROM_CACHE, java.lang.Boolean.FALSE) - TaskUtils.addRuntimeMap(job.getParams, runtime) - val groupName = schedulerContext.getOrCreateGroupFactory.getOrCreateGroup(job).getGroupName - val consumer = schedulerContext.getOrCreateConsumerManager.getOrCreateConsumer(groupName) - val index = consumer.getConsumeQueue.offer(job) - // index.map(getEventId(_, groupName)).foreach(job.setId) - if (index.isEmpty) { - throw new SchedulerErrorException( - JOB_QUEUE_IS_FULL.getErrorCode, - JOB_QUEUE_IS_FULL.getErrorDesc - ) - } - } - -} diff --git a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/scheduler/cache/ReadCacheConsumerManager.scala b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/scheduler/cache/ReadCacheConsumerManager.scala deleted file mode 100644 index a4cba19f34..0000000000 --- a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/scheduler/cache/ReadCacheConsumerManager.scala +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.entrance.scheduler.cache - -import org.apache.linkis.entrance.persistence.PersistenceManager -import org.apache.linkis.entrance.scheduler.EntranceGroupFactory -import org.apache.linkis.scheduler.queue.fifoqueue.FIFOUserConsumer -import org.apache.linkis.scheduler.queue.parallelqueue.ParallelConsumerManager - -class ReadCacheConsumerManager(maxParallelismUsers: Int, persistenceManager: PersistenceManager) - extends ParallelConsumerManager(maxParallelismUsers) { - - override protected def createConsumer(groupName: String): FIFOUserConsumer = { - val group = getSchedulerContext.getOrCreateGroupFactory.getGroup(groupName) - if (groupName.endsWith(EntranceGroupFactory.CACHE)) { - logger.info("Create cache consumer with group: " + groupName) - new ReadCacheConsumer( - getSchedulerContext, - getOrCreateExecutorService, - group, - persistenceManager - ) - } else { - logger.info("Create normal consumer with group: " + groupName) - new FIFOUserConsumer(getSchedulerContext, getOrCreateExecutorService, group) - } - } - -} diff --git a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/utils/EntranceUtils.scala b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/utils/EntranceUtils.scala new file mode 100644 index 0000000000..13dcefa9f9 --- /dev/null +++ b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/utils/EntranceUtils.scala @@ -0,0 +1,108 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.entrance.utils + +import org.apache.linkis.common.ServiceInstance +import org.apache.linkis.common.utils.{Logging, Utils} +import org.apache.linkis.instance.label.client.InstanceLabelClient +import org.apache.linkis.manager.label.builder.factory.LabelBuilderFactoryContext +import org.apache.linkis.manager.label.constant.{LabelKeyConstant, LabelValueConstant} +import org.apache.linkis.manager.label.entity.Label +import org.apache.linkis.manager.label.entity.engine.{EngineTypeLabel, UserCreatorLabel} +import org.apache.linkis.manager.label.entity.route.RouteLabel +import org.apache.linkis.manager.label.utils.EngineTypeLabelCreator +import org.apache.linkis.rpc.Sender + +import org.apache.commons.lang3.StringUtils + +import java.util + +import scala.collection.JavaConverters.asScalaBufferConverter + +object EntranceUtils extends Logging { + + private val SPLIT = "," + + private val labelFactory = LabelBuilderFactoryContext.getLabelBuilderFactory + + def getUserCreatorEcTypeKey( + userCreatorLabel: UserCreatorLabel, + engineTypeLabel: EngineTypeLabel + ): String = { + userCreatorLabel.getStringValue + SPLIT + engineTypeLabel.getStringValue + } + + def fromKeyGetLabels(key: String): (UserCreatorLabel, EngineTypeLabel) = { + if (StringUtils.isBlank(key)) (null, null) + else { + val labelStringValues = key.split(SPLIT) + if (labelStringValues.length < 2) return (null, null) + val userCreatorLabel = labelFactory + .createLabel[UserCreatorLabel](LabelKeyConstant.USER_CREATOR_TYPE_KEY, labelStringValues(0)) + val engineTypeLabel = labelFactory + .createLabel[EngineTypeLabel](LabelKeyConstant.ENGINE_TYPE_KEY, labelStringValues(1)) + (userCreatorLabel, engineTypeLabel) + } + } + + def getDefaultCreatorECTypeKey(creator: String, ecType: String): String = { + val userCreatorLabel = + labelFactory.createLabel[UserCreatorLabel](LabelKeyConstant.USER_CREATOR_TYPE_KEY) + val ecTypeLabel = EngineTypeLabelCreator.createEngineTypeLabel(ecType) + userCreatorLabel.setUser("*") + userCreatorLabel.setCreator(creator) + getUserCreatorEcTypeKey(userCreatorLabel, ecTypeLabel) + } + + def getRunningEntranceNumber(): Int = { + val entranceNum = Sender.getInstances(Sender.getThisServiceInstance.getApplicationName).length + val labelList = new util.ArrayList[Label[_]]() + val offlineRouteLabel = LabelBuilderFactoryContext.getLabelBuilderFactory + .createLabel[RouteLabel](LabelKeyConstant.ROUTE_KEY, LabelValueConstant.OFFLINE_VALUE) + labelList.add(offlineRouteLabel) + var offlineIns: Array[ServiceInstance] = null + Utils.tryAndWarn { + offlineIns = InstanceLabelClient.getInstance + .getInstanceFromLabel(labelList) + .asScala + .filter(l => + null != l && l.getApplicationName + .equalsIgnoreCase(Sender.getThisServiceInstance.getApplicationName) + ) + .toArray + } + val entranceRealNumber = if (null != offlineIns) { + logger.info(s"There are ${offlineIns.length} offlining instance.") + entranceNum - offlineIns.length + } else { + entranceNum + } + /* + Sender.getInstances may get 0 instances due to cache in Sender. So this instance is the one instance. + */ + if (entranceRealNumber <= 0) { + logger.error( + s"Got ${entranceRealNumber} ${Sender.getThisServiceInstance.getApplicationName} instances." + ) + 1 + } else { + entranceRealNumber + } + } + +} diff --git a/linkis-public-enhancements/linkis-instance-label/linkis-instance-label-client/src/main/scala/org/apache/linkis/instance/label/client/InstanceLabelClient.scala b/linkis-public-enhancements/linkis-instance-label/linkis-instance-label-client/src/main/scala/org/apache/linkis/instance/label/client/InstanceLabelClient.scala index efaf43818c..677898352a 100644 --- a/linkis-public-enhancements/linkis-instance-label/linkis-instance-label-client/src/main/scala/org/apache/linkis/instance/label/client/InstanceLabelClient.scala +++ b/linkis-public-enhancements/linkis-instance-label/linkis-instance-label-client/src/main/scala/org/apache/linkis/instance/label/client/InstanceLabelClient.scala @@ -62,41 +62,36 @@ class InstanceLabelClient extends Logging { def getLabelFromInstance(serviceInstance: ServiceInstance): util.List[Label[_]] = { val request = new InsLabelQueryRequest(serviceInstance) - Utils.tryAndError { - getSender().ask(request) match { + Utils.tryAndWarn { + val respObj = getSender().ask(request) + respObj match { case resp: InsLabelQueryResponse => val labelList = new util.ArrayList[Label[_]]() resp.getLabelList.asScala.foreach(pair => labelList.add(labelBuilderFactory.createLabel[Label[_]](pair.getKey, pair.getValue)) ) labelList - case o => - logger.error(s"Invalid response ${BDPJettyServerHelper.gson - .toJson(o)} from request : ${BDPJettyServerHelper.gson.toJson(request)}") + case _ => + logger.warn(s"Invalid resp :$respObj from request : $request") new util.ArrayList[Label[_]] } } } def getInstanceFromLabel(labels: util.List[Label[_]]): util.List[ServiceInstance] = { - Utils.tryAndError { + Utils.tryAndWarn { val request = new LabelInsQueryRequest() val labelMap = LabelUtils.labelsToMap(labels) request.setLabels(labelMap.asInstanceOf[util.HashMap[String, Object]]) - Sender.getSender(PUBLIC_SERVICE_APPLICATION_NAME.getValue).ask(request) match { + val respObj = getSender().ask(request) + respObj match { case resp: LabelInsQueryResponse => if (null == resp.getInsList || resp.getInsList.isEmpty) { return new util.ArrayList[ServiceInstance]() } - if (resp.getInsList.size() != 1) { - logger.warn( - s"Instance num ${resp.getInsList.size()} with labels ${BDPJettyServerHelper.gson.toJson(labelMap)} is not single one." - ) - } resp.getInsList - case o => - logger.error(s"Invalid resp : ${JsonUtils.jackson - .writeValueAsString(o)} from request : ${BDPJettyServerHelper.gson.toJson(request)}") + case _ => + logger.warn(s"Invalid resp :$respObj from request : $request") new util.ArrayList[ServiceInstance]() } } From 77f06146bf18f8d6828cf5e69280d67578326c3f Mon Sep 17 00:00:00 2001 From: peacewong Date: Tue, 23 May 2023 11:09:50 +0800 Subject: [PATCH 120/261] remove CacheOutputExecuteResponse --- .../persistence/EntranceResultSetEngine.scala | 28 ++++++------------- 1 file changed, 8 insertions(+), 20 deletions(-) diff --git a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/persistence/EntranceResultSetEngine.scala b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/persistence/EntranceResultSetEngine.scala index 111350806e..2ba98438e8 100644 --- a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/persistence/EntranceResultSetEngine.scala +++ b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/persistence/EntranceResultSetEngine.scala @@ -17,20 +17,12 @@ package org.apache.linkis.entrance.persistence -import org.apache.linkis.common.io.{FsPath, MetaData, Record} -import org.apache.linkis.common.io.resultset.ResultSet -import org.apache.linkis.common.utils.{Logging, Utils} +import org.apache.linkis.common.utils.Logging import org.apache.linkis.entrance.exception.{EntranceErrorCode, EntranceErrorException} -import org.apache.linkis.entrance.execute.StorePathExecuteRequest -import org.apache.linkis.entrance.job.{EntranceExecuteRequest, EntranceExecutionJob} -import org.apache.linkis.entrance.scheduler.cache.CacheOutputExecuteResponse -import org.apache.linkis.governance.common.entity.job.SubJobDetail import org.apache.linkis.scheduler.executer.{AliasOutputExecuteResponse, OutputExecuteResponse} import org.apache.linkis.scheduler.queue.Job -import org.apache.linkis.storage.resultset.{ResultSetFactory, ResultSetWriter} -import org.apache.linkis.storage.utils.FileSystemUtils +import org.apache.linkis.storage.resultset.ResultSetFactory -import org.apache.commons.io.IOUtils import org.apache.commons.lang3.StringUtils class EntranceResultSetEngine extends ResultSetEngine with Logging { @@ -46,15 +38,11 @@ class EntranceResultSetEngine extends ResultSetEngine with Logging { EntranceErrorCode.RESULT_NOT_PERSISTED_ERROR.getDesc ) } - case CacheOutputExecuteResponse(alias, output) => - if (ResultSetFactory.getInstance.isResultSetPath(output)) { - getDir(output) - } else { - throw new EntranceErrorException( - EntranceErrorCode.RESULT_NOT_PERSISTED_ERROR.getErrCode, - EntranceErrorCode.RESULT_NOT_PERSISTED_ERROR.getDesc - ) - } + case _ => + throw new EntranceErrorException( + EntranceErrorCode.RESULT_NOT_PERSISTED_ERROR.getErrCode, + EntranceErrorCode.RESULT_NOT_PERSISTED_ERROR.getDesc + ) } } @@ -64,7 +52,7 @@ class EntranceResultSetEngine extends ResultSetEngine with Logging { } else { val arr = str.split("/").filter(StringUtils.isNotBlank) if (arr.length <= 2) { - return str + str } else { str.substring(0, str.lastIndexOf("/")) } From 5d2ce9416a664dcaf0151ba3df9611ac3798e0a3 Mon Sep 17 00:00:00 2001 From: peacewong Date: Tue, 23 May 2023 18:15:11 +0800 Subject: [PATCH 121/261] Fix build error --- .../executor/conf/AccessibleExecutorConfiguration.scala | 2 +- .../apache/linkis/entrance/server/DefaultEntranceServer.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/org/apache/linkis/engineconn/acessible/executor/conf/AccessibleExecutorConfiguration.scala b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/org/apache/linkis/engineconn/acessible/executor/conf/AccessibleExecutorConfiguration.scala index 26a25a1539..f891df65f4 100644 --- a/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/org/apache/linkis/engineconn/acessible/executor/conf/AccessibleExecutorConfiguration.scala +++ b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/org/apache/linkis/engineconn/acessible/executor/conf/AccessibleExecutorConfiguration.scala @@ -37,7 +37,7 @@ object AccessibleExecutorConfiguration { val ENGINECONN_LOG_SEND_SIZE = CommonVars[Int]("wds.linkis.engineconn.log.send.cache.size", 300) val ENGINECONN_MAX_FREE_TIME = - CommonVars("wds.linkis.engineconn.max.free.time", new TimeType("30m")) + CommonVars("wds.linkis.engineconn.max.free.time", new TimeType("5m")) val ENGINECONN_LOCK_CHECK_INTERVAL = CommonVars("wds.linkis.engineconn.lock.free.interval", new TimeType("3m")) diff --git a/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/server/DefaultEntranceServer.java b/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/server/DefaultEntranceServer.java index ddf8e45767..fac85f32c1 100644 --- a/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/server/DefaultEntranceServer.java +++ b/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/server/DefaultEntranceServer.java @@ -94,7 +94,7 @@ private void shutdownEntrance(ContextClosedEvent event) { logger.warn("event has been handled"); } else { logger.warn("Entrance exit to stop all job"); - EntranceJob[] allUndoneJobs = getAllUndoneTask(null); + EntranceJob[] allUndoneJobs = getAllUndoneTask(null, null); if (null != allUndoneJobs) { for (EntranceJob job : allUndoneJobs) { job.onFailure( From 937f722fcdfbab99b6c654ddb33ff2eaa407bafe Mon Sep 17 00:00:00 2001 From: HmhWz <934178858@qq.com> Date: Tue, 23 May 2023 21:18:09 +0800 Subject: [PATCH 122/261] The dss prod label interface forwarding problem was fixed. --- .../apache/linkis/gateway/dss/parser/DSSGatewayParser.scala | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/linkis-spring-cloud-services/linkis-service-gateway/linkis-gateway-server-support/src/main/scala/org/apache/linkis/gateway/dss/parser/DSSGatewayParser.scala b/linkis-spring-cloud-services/linkis-service-gateway/linkis-gateway-server-support/src/main/scala/org/apache/linkis/gateway/dss/parser/DSSGatewayParser.scala index 10e8fceda0..e1b4bf430e 100644 --- a/linkis-spring-cloud-services/linkis-service-gateway/linkis-gateway-server-support/src/main/scala/org/apache/linkis/gateway/dss/parser/DSSGatewayParser.scala +++ b/linkis-spring-cloud-services/linkis-service-gateway/linkis-gateway-server-support/src/main/scala/org/apache/linkis/gateway/dss/parser/DSSGatewayParser.scala @@ -266,8 +266,10 @@ class DSSGatewayParser extends AbstractGatewayParser { retService } } - val lowerServiceId = parsedServiceId.toLowerCase(Locale.getDefault()) + var lowerServiceId = parsedServiceId.toLowerCase(Locale.getDefault()) val serverName = tmpServerName.toLowerCase(Locale.getDefault()) + // 让prod的接口匹配到prod的服务 + if (serverName.endsWith("-prod")) lowerServiceId += "/prod" findIt(_.toLowerCase(Locale.getDefault()) == serverName).orElse(findMostCorrect(service => { (service, lowerServiceId.split("/").count(word => service.contains(word))) })) From c51249a56e109df16e6fd7cd0283548d8466908d Mon Sep 17 00:00:00 2001 From: peacewong Date: Mon, 29 May 2023 15:47:45 +0800 Subject: [PATCH 123/261] add scheduler conf --- .../linkis/entrance/scheduler/CreatorECTypeDefaultConf.java | 4 +++- .../apache/linkis/entrance/conf/EntranceConfiguration.scala | 3 +++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/scheduler/CreatorECTypeDefaultConf.java b/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/scheduler/CreatorECTypeDefaultConf.java index 8a28a5e92f..5a91c71a11 100644 --- a/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/scheduler/CreatorECTypeDefaultConf.java +++ b/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/scheduler/CreatorECTypeDefaultConf.java @@ -49,7 +49,9 @@ public class CreatorECTypeDefaultConf { private static LoadingCache confCache = CacheBuilder.newBuilder() .maximumSize(1000) - .expireAfterWrite(1, TimeUnit.HOURS) + .expireAfterWrite( + (long) EntranceConfiguration.ENTRANCE_CREATOR_JOB_LIMIT_CONF_CACHE().getValue(), + TimeUnit.MINUTES) .build( new CacheLoader() { @Override diff --git a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/conf/EntranceConfiguration.scala b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/conf/EntranceConfiguration.scala index b7380896f8..63e030e389 100644 --- a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/conf/EntranceConfiguration.scala +++ b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/conf/EntranceConfiguration.scala @@ -230,4 +230,7 @@ object EntranceConfiguration { "Creator task concurrency limit parameters" ) + val ENTRANCE_CREATOR_JOB_LIMIT_CONF_CACHE = + CommonVars("linkis.entrance.creator.job.concurrency.limit.conf.cache.time", 30L) + } From 11ca3e6492ff1fb2d24020f7fcb6cfba90741fc8 Mon Sep 17 00:00:00 2001 From: CharlieYan <16526279+charlieyan24@users.noreply.github.com> Date: Tue, 25 Apr 2023 21:25:12 +0800 Subject: [PATCH 124/261] feat: do not kill ec when ecm restart part-1 (#4184) (#4452) * feat: do not kill ec when ecm restart(#4192) --- .../callback/LogCallbackProtocol.scala | 3 - .../common/conf/GovernaceCommonConf.scala | 8 +- .../governance/common/utils/ECPathUtils.scala | 18 + .../common/utils/GovernanceUtils.scala | 20 + .../ecm/core/report/NodeHealthReport.scala | 4 +- .../ecm/server/listener/ECMReadyEvent.scala | 14 +- .../operator/EngineConnLogOperator.scala | 58 +-- .../report/DefaultECMHealthReport.scala | 4 +- .../service/EngineConnListService.scala | 47 -- .../EngineConnStatusCallbackService.scala | 26 - .../server/service/LogCallbackService.scala | 26 - .../AbstractEngineConnLaunchService.scala | 8 +- .../impl/DefaultECMHealthService.scala | 9 +- .../impl/DefaultEngineConnKillService.java | 99 ++-- .../impl/DefaultEngineConnListService.scala | 192 -------- .../DefaultEngineConnPidCallbackService.scala | 39 -- ...faultEngineConnStatusCallbackService.scala | 67 --- .../impl/DefaultYarnCallbackService.scala | 40 -- .../impl/ProcessEngineConnLaunchService.scala | 6 +- .../spring/ECMSpringConfiguration.scala | 28 -- .../ECTaskEntranceInfoAccessHelper.scala | 96 ---- .../engineconn/launch/EngineConnServer.scala | 1 + .../hook/CallbackEngineConnHook.scala | 10 +- .../EngineConnAfterStartCallback.scala | 5 +- .../callback/service/EngineConnCallback.scala | 20 +- .../service/EngineConnPidCallback.scala | 4 +- .../{am => }/LinkisManagerApplication.java | 2 +- .../message/EngineConnPidCallbackEvent.java} | 18 +- .../manager/am/restful/EMRestfulApi.java | 31 +- .../EngineConnPidCallbackService.java} | 11 +- .../EngineConnStatusCallbackService.java} | 9 +- .../DefaultEngineConnPidCallbackService.java | 61 +++ ...efaultEngineConnStatusCallbackService.java | 146 ++++++ .../am/manager/DefaultEngineNodeManager.scala | 12 +- .../service/em/DefaultEMEngineService.scala | 63 +++ ...faultEngineConnStatusCallbackService.scala | 104 ---- .../engine/DefaultEngineStopService.scala | 6 +- .../am/service/engine/EngineStopService.scala | 5 - .../manager/rm/restful/RMMonitorRest.scala | 455 ++++++++++-------- .../rm/service/impl/ResourceLogService.scala | 26 +- .../manager/common/entity/node/AMEMNode.java | 11 + .../common/entity/node/AMEngineNode.java | 11 + .../common/entity/node/InfoRMNode.java | 11 + .../manager/common/entity/node/Node.java | 4 + .../entity/persistence/PersistenceNode.java | 10 + .../persistence/PersistenceNodeEntity.java | 11 + .../persistence/PersistenceNodeMetrics.java | 5 +- .../protocol/engine/EngineStopRequest.java | 51 +- .../linkis/manager/dao/NodeManagerMapper.java | 2 +- .../NodeMetricManagerPersistence.java | 2 +- .../impl/DefaultNodeManagerPersistence.java | 12 +- .../DefaultNodeMetricManagerPersistence.java | 2 +- .../mapper/common/NodeManagerMapper.xml | 109 ++++- .../manager/dao/NodeManagerMapperTest.java | 2 +- .../linkis/templates/configmap-init-sql.yaml | 5 +- linkis-dist/package/db/linkis_ddl.sql | 1 + .../package/db/module/linkis_manager.sql | 1 + .../package/sbin/ext/linkis-cg-linkismanager | 2 +- 58 files changed, 921 insertions(+), 1132 deletions(-) delete mode 100644 linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/service/EngineConnListService.scala delete mode 100644 linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/service/EngineConnStatusCallbackService.scala delete mode 100644 linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/service/LogCallbackService.scala delete mode 100644 linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/service/impl/DefaultEngineConnListService.scala delete mode 100644 linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/service/impl/DefaultEngineConnPidCallbackService.scala delete mode 100644 linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/service/impl/DefaultEngineConnStatusCallbackService.scala delete mode 100644 linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/service/impl/DefaultYarnCallbackService.scala delete mode 100644 linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/test/scala/org/apache/linkis/engineconn/computation/executor/upstream/access/ECTaskEntranceInfoAccessHelper.scala rename linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/{am => }/LinkisManagerApplication.java (96%) rename linkis-computation-governance/{linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/service/YarnCallbackService.scala => linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/event/message/EngineConnPidCallbackEvent.java} (62%) rename linkis-computation-governance/{linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/service/EngineConnPidCallbackService.scala => linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/service/EngineConnPidCallbackService.java} (82%) rename linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/{scala/org/apache/linkis/manager/am/service/engine/EngineConnStatusCallbackService.scala => java/org/apache/linkis/manager/am/service/EngineConnStatusCallbackService.java} (79%) create mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/service/impl/DefaultEngineConnPidCallbackService.java create mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/service/impl/DefaultEngineConnStatusCallbackService.java delete mode 100644 linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/service/engine/DefaultEngineConnStatusCallbackService.scala diff --git a/linkis-commons/linkis-protocol/src/main/scala/org/apache/linkis/protocol/callback/LogCallbackProtocol.scala b/linkis-commons/linkis-protocol/src/main/scala/org/apache/linkis/protocol/callback/LogCallbackProtocol.scala index 91b0e01727..0109472a90 100644 --- a/linkis-commons/linkis-protocol/src/main/scala/org/apache/linkis/protocol/callback/LogCallbackProtocol.scala +++ b/linkis-commons/linkis-protocol/src/main/scala/org/apache/linkis/protocol/callback/LogCallbackProtocol.scala @@ -19,9 +19,6 @@ package org.apache.linkis.protocol.callback import org.apache.linkis.protocol.message.RequestProtocol -// TODO: log type -case class LogCallbackProtocol(nodeId: String, logs: Array[String]) extends RequestProtocol - case class YarnAPPIdCallbackProtocol(nodeId: String, applicationId: String) extends RequestProtocol case class YarnInfoCallbackProtocol(nodeId: String, uri: String) extends RequestProtocol diff --git a/linkis-computation-governance/linkis-computation-governance-common/src/main/scala/org/apache/linkis/governance/common/conf/GovernaceCommonConf.scala b/linkis-computation-governance/linkis-computation-governance-common/src/main/scala/org/apache/linkis/governance/common/conf/GovernaceCommonConf.scala index 780e50878f..4838c7d21d 100644 --- a/linkis-computation-governance/linkis-computation-governance-common/src/main/scala/org/apache/linkis/governance/common/conf/GovernaceCommonConf.scala +++ b/linkis-computation-governance/linkis-computation-governance-common/src/main/scala/org/apache/linkis/governance/common/conf/GovernaceCommonConf.scala @@ -40,10 +40,16 @@ object GovernanceCommonConf { val ENGINE_CONN_MANAGER_SPRING_NAME = CommonVars("wds.linkis.engineconn.manager.name", "linkis-cg-engineconnmanager") + val ENGINE_APPLICATION_MANAGER_SPRING_NAME = + CommonVars("wds.linkis.application.manager.name", "linkis-cg-linkismanager") + val ENGINE_CONN_PORT_RANGE = CommonVars("wds.linkis.engineconn.port.range", "-") val MANAGER_SERVICE_NAME = - CommonVars("wds.linkis.engineconn.manager.name", "linkis-cg-linkismanager") + CommonVars( + "wds.linkis.engineconn.manager.name", + GovernanceCommonConf.ENGINE_APPLICATION_MANAGER_SPRING_NAME.getValue + ) val ENTRANCE_SERVICE_NAME = CommonVars("wds.linkis.entrance.name", "linkis-cg-entrance") diff --git a/linkis-computation-governance/linkis-computation-governance-common/src/main/scala/org/apache/linkis/governance/common/utils/ECPathUtils.scala b/linkis-computation-governance/linkis-computation-governance-common/src/main/scala/org/apache/linkis/governance/common/utils/ECPathUtils.scala index 58c08b1f84..236046f3d4 100644 --- a/linkis-computation-governance/linkis-computation-governance-common/src/main/scala/org/apache/linkis/governance/common/utils/ECPathUtils.scala +++ b/linkis-computation-governance/linkis-computation-governance-common/src/main/scala/org/apache/linkis/governance/common/utils/ECPathUtils.scala @@ -17,6 +17,8 @@ package org.apache.linkis.governance.common.utils +import org.apache.linkis.manager.label.entity.engine.{EngineTypeLabel, UserCreatorLabel} + import org.apache.commons.lang3.StringUtils import org.apache.commons.lang3.time.DateFormatUtils @@ -45,4 +47,20 @@ object ECPathUtils { suffix + File.separator + ticketId } + def getECLogDirSuffix( + engineTypeLabel: EngineTypeLabel, + userCreatorLabel: UserCreatorLabel, + ticketId: String + ): String = { + if (null == engineTypeLabel || null == userCreatorLabel) { + return "" + } + val suffix = ECPathUtils.getECWOrkDirPathSuffix( + userCreatorLabel.getUser, + ticketId, + engineTypeLabel.getEngineType + ) + suffix + File.separator + "logs" + } + } diff --git a/linkis-computation-governance/linkis-computation-governance-common/src/main/scala/org/apache/linkis/governance/common/utils/GovernanceUtils.scala b/linkis-computation-governance/linkis-computation-governance-common/src/main/scala/org/apache/linkis/governance/common/utils/GovernanceUtils.scala index 43fd598f71..301e295ef1 100644 --- a/linkis-computation-governance/linkis-computation-governance-common/src/main/scala/org/apache/linkis/governance/common/utils/GovernanceUtils.scala +++ b/linkis-computation-governance/linkis-computation-governance-common/src/main/scala/org/apache/linkis/governance/common/utils/GovernanceUtils.scala @@ -25,6 +25,7 @@ import org.apache.commons.lang3.StringUtils import java.io.File import java.util +import java.util.{ArrayList, List} object GovernanceUtils extends Logging { @@ -72,4 +73,23 @@ object GovernanceUtils extends Logging { } } + /** + * find process id by port number + * @param processPort + * @return + */ + def findProcessIdentifier(processPort: String) = { + val findCmd = "sudo lsof -t -i:" + processPort + val cmdList = new util.ArrayList[String] + cmdList.add("bash") + cmdList.add("-c") + cmdList.add(findCmd) + try Utils.exec(cmdList.toArray(new Array[String](0)), 5000L) + catch { + case e: Exception => + logger.warn("Method findPid failed, " + e.getMessage) + null + } + } + } diff --git a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-core/src/main/scala/org/apache/linkis/ecm/core/report/NodeHealthReport.scala b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-core/src/main/scala/org/apache/linkis/ecm/core/report/NodeHealthReport.scala index 160025ed51..395c9258b8 100644 --- a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-core/src/main/scala/org/apache/linkis/ecm/core/report/NodeHealthReport.scala +++ b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-core/src/main/scala/org/apache/linkis/ecm/core/report/NodeHealthReport.scala @@ -34,9 +34,9 @@ trait NodeHealthReport { def getNodeMsg: String - def getUsedResource: Resource +// def getUsedResource: Resource - def setUsedResource(resource: Resource): Unit +// def setUsedResource(resource: Resource): Unit def getTotalResource: Resource diff --git a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/listener/ECMReadyEvent.scala b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/listener/ECMReadyEvent.scala index 6bcb6c3b8b..97243d3cc0 100644 --- a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/listener/ECMReadyEvent.scala +++ b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/listener/ECMReadyEvent.scala @@ -17,7 +17,6 @@ package org.apache.linkis.ecm.server.listener -import org.apache.linkis.ecm.core.engineconn.EngineConn import org.apache.linkis.ecm.core.listener.ECMEvent import org.apache.linkis.governance.common.protocol.task.ResponseEngineConnPid import org.apache.linkis.manager.common.entity.enumeration.NodeStatus @@ -27,14 +26,5 @@ case class ECMReadyEvent(params: Array[String]) extends ECMEvent case class ECMClosedEvent() extends ECMEvent -case class EngineConnStatusChageEvent(from: NodeStatus, to: NodeStatus) extends ECMEvent - -case class YarnAppIdCallbackEvent(protocol: YarnAPPIdCallbackProtocol) extends ECMEvent - -case class YarnInfoCallbackEvent(protocol: YarnInfoCallbackProtocol) extends ECMEvent - -case class EngineConnPidCallbackEvent(protocol: ResponseEngineConnPid) extends ECMEvent - -case class EngineConnAddEvent(conn: EngineConn) extends ECMEvent - -case class EngineConnStatusChangeEvent(tickedId: String, updateStatus: NodeStatus) extends ECMEvent +case class EngineConnLaunchStatusChangeEvent(tickedId: String, updateStatus: NodeStatus) + extends ECMEvent diff --git a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/operator/EngineConnLogOperator.scala b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/operator/EngineConnLogOperator.scala index b558c319cd..dd7f17356b 100644 --- a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/operator/EngineConnLogOperator.scala +++ b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/operator/EngineConnLogOperator.scala @@ -17,15 +17,13 @@ package org.apache.linkis.ecm.server.operator -import org.apache.linkis.DataWorkCloudApplication import org.apache.linkis.common.conf.CommonVars import org.apache.linkis.common.utils.{Logging, Utils} import org.apache.linkis.ecm.errorcode.EngineconnServerErrorCodeSummary._ import org.apache.linkis.ecm.server.conf.ECMConfiguration import org.apache.linkis.ecm.server.exception.ECMErrorException -import org.apache.linkis.ecm.server.service.{EngineConnListService, LocalDirsHandleService} +import org.apache.linkis.ecm.server.service.LocalDirsHandleService import org.apache.linkis.manager.common.operator.Operator -import org.apache.linkis.manager.common.protocol.em.ECMOperateRequest import org.apache.commons.io.IOUtils import org.apache.commons.io.input.ReversedLinesFileReader @@ -37,12 +35,10 @@ import java.text.MessageFormat import java.util import java.util.Collections -import scala.collection.JavaConverters.asScalaBufferConverter import scala.util.matching.Regex class EngineConnLogOperator extends Operator with Logging { - private var engineConnListService: EngineConnListService = _ private var localDirsHandleService: LocalDirsHandleService = _ override def getNames: Array[String] = Array(EngineConnLogOperator.OPERATOR_NAME) @@ -162,56 +158,10 @@ class EngineConnLogOperator extends Operator with Logging { protected def getEngineConnInfo(implicit parameters: Map[String, Any] ): (String, String, String) = { - if (engineConnListService == null) { - engineConnListService = - DataWorkCloudApplication.getApplicationContext.getBean(classOf[EngineConnListService]) - localDirsHandleService = - DataWorkCloudApplication.getApplicationContext.getBean(classOf[LocalDirsHandleService]) - } val logDIrSuffix = getAs("logDirSuffix", "") - val (engineConnLogDir, engineConnInstance, ticketId) = - if (StringUtils.isNotBlank(logDIrSuffix)) { - val ecLogPath = ECMConfiguration.ENGINECONN_ROOT_DIR + File.separator + logDIrSuffix - val ticketId = getAs("ticketId", "") - (ecLogPath, "", ticketId) - } else { - val engineConnInstance = getAs( - ECMOperateRequest.ENGINE_CONN_INSTANCE_KEY, - getAs[String]("engineConnInstance", null) - ) - Option(engineConnInstance) - .flatMap { instance => - engineConnListService.getEngineConns.asScala.find( - _.getServiceInstance.getInstance == instance - ) - } - .map(engineConn => - ( - engineConn.getEngineConnManagerEnv.engineConnLogDirs, - engineConnInstance, - engineConn.getTickedId - ) - ) - .getOrElse { - val ticketId = getAs("ticketId", "") - if (StringUtils.isBlank(ticketId)) { - throw new ECMErrorException( - BOTH_NOT_EXISTS.getErrorCode, - s"the parameters of ${ECMOperateRequest.ENGINE_CONN_INSTANCE_KEY}, engineConnInstance and ticketId are both not exists." - ) - } - val logDir = engineConnListService - .getEngineConn(ticketId) - .map(_.getEngineConnManagerEnv.engineConnLogDirs) - .getOrElse { - val creator = getAsThrow[String]("creator") - val engineConnType = getAsThrow[String]("engineConnType") - localDirsHandleService.getEngineConnLogDir(creator, ticketId, engineConnType) - } - (logDir, engineConnInstance, ticketId) - } - } - (ticketId, engineConnInstance, engineConnLogDir) + val ecLogPath = ECMConfiguration.ENGINECONN_ROOT_DIR + File.separator + logDIrSuffix + val ticketId = getAs("ticketId", "") + (ticketId, "", ecLogPath) } private def includeLine( diff --git a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/report/DefaultECMHealthReport.scala b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/report/DefaultECMHealthReport.scala index 6fdf8b3488..150d0be6ba 100644 --- a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/report/DefaultECMHealthReport.scala +++ b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/report/DefaultECMHealthReport.scala @@ -69,9 +69,9 @@ class DefaultECMHealthReport extends ECMHealthReport { override def getNodeMsg: String = nodeMsg - override def getUsedResource: Resource = usedResource +// override def getUsedResource: Resource = usedResource - override def setUsedResource(usedResource: Resource): Unit = this.usedResource = usedResource +// override def setUsedResource(usedResource: Resource): Unit = this.usedResource = usedResource override def getTotalResource: Resource = totalResource diff --git a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/service/EngineConnListService.scala b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/service/EngineConnListService.scala deleted file mode 100644 index d36d35cceb..0000000000 --- a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/service/EngineConnListService.scala +++ /dev/null @@ -1,47 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.ecm.server.service - -import org.apache.linkis.ecm.core.engineconn.EngineConn -import org.apache.linkis.ecm.core.launch.EngineConnLaunchRunner -import org.apache.linkis.manager.common.entity.resource.Resource - -import java.util - -/** - * The enginelistservice interface manages the interface started by the engine The most important - * submit method is to put the thread that starts the engine into the thread pool to start - * EngineListService接口管理引擎启动的接口 最重要的submit方法是将启动引擎的线程放入到线程池中进行启动 - */ -trait EngineConnListService { - - def init(): Unit - - def getEngineConn(engineConnId: String): Option[EngineConn] - - def getEngineConns: util.List[EngineConn] - - def addEngineConn(engineConn: EngineConn): Unit - - def killEngineConn(engineConnId: String): Unit - - def getUsedResources: Resource - - def submit(runner: EngineConnLaunchRunner): Option[EngineConn] - -} diff --git a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/service/EngineConnStatusCallbackService.scala b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/service/EngineConnStatusCallbackService.scala deleted file mode 100644 index 12ccc088be..0000000000 --- a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/service/EngineConnStatusCallbackService.scala +++ /dev/null @@ -1,26 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.ecm.server.service - -import org.apache.linkis.manager.common.protocol.engine.EngineConnStatusCallback - -trait EngineConnStatusCallbackService { - - def dealEngineConnStatusCallback(protocol: EngineConnStatusCallback): Unit - -} diff --git a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/service/LogCallbackService.scala b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/service/LogCallbackService.scala deleted file mode 100644 index 873b098194..0000000000 --- a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/service/LogCallbackService.scala +++ /dev/null @@ -1,26 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.ecm.server.service - -import org.apache.linkis.protocol.callback.LogCallbackProtocol - -trait LogCallbackService { - - def dealLog(protocol: LogCallbackProtocol): Unit - -} diff --git a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/service/impl/AbstractEngineConnLaunchService.scala b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/service/impl/AbstractEngineConnLaunchService.scala index c6c598a2bb..19581c99fe 100644 --- a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/service/impl/AbstractEngineConnLaunchService.scala +++ b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/service/impl/AbstractEngineConnLaunchService.scala @@ -25,7 +25,7 @@ import org.apache.linkis.ecm.server.LinkisECMApplication import org.apache.linkis.ecm.server.conf.ECMConfiguration._ import org.apache.linkis.ecm.server.engineConn.DefaultEngineConn import org.apache.linkis.ecm.server.hook.ECMHook -import org.apache.linkis.ecm.server.listener.{EngineConnAddEvent, EngineConnStatusChangeEvent} +import org.apache.linkis.ecm.server.listener.EngineConnLaunchStatusChangeEvent import org.apache.linkis.ecm.server.service.{EngineConnLaunchService, ResourceLocalizationService} import org.apache.linkis.ecm.server.util.ECMUtils import org.apache.linkis.governance.common.conf.GovernanceCommonConf @@ -80,8 +80,6 @@ abstract class AbstractEngineConnLaunchService extends EngineConnLaunchService w conn.setEngineConnManagerEnv(launch.getEngineConnManagerEnv()) // 2.资源本地化，并且设置ecm的env环境信息 getResourceLocalizationServie.handleInitEngineConnResources(request, conn) - // 3.添加到list - LinkisECMApplication.getContext.getECMSyncListenerBus.postToAll(EngineConnAddEvent(conn)) // 4.run Utils.tryCatch { beforeLaunch(request, conn, duration) @@ -123,7 +121,7 @@ abstract class AbstractEngineConnLaunchService extends EngineConnLaunchService w ): _* ) LinkisECMApplication.getContext.getECMSyncListenerBus.postToAll( - EngineConnStatusChangeEvent(conn.getTickedId, Failed) + EngineConnLaunchStatusChangeEvent(conn.getTickedId, Failed) ) LoggerUtils.removeJobIdMDC() case Success(_) => @@ -166,7 +164,7 @@ abstract class AbstractEngineConnLaunchService extends EngineConnLaunchService w ) ) LinkisECMApplication.getContext.getECMSyncListenerBus.postToAll( - EngineConnStatusChangeEvent(conn.getTickedId, Failed) + EngineConnLaunchStatusChangeEvent(conn.getTickedId, Failed) ) LoggerUtils.removeJobIdMDC() throw t diff --git a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/service/impl/DefaultECMHealthService.scala b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/service/impl/DefaultECMHealthService.scala index 132749cbec..be879f6877 100644 --- a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/service/impl/DefaultECMHealthService.scala +++ b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/service/impl/DefaultECMHealthService.scala @@ -25,7 +25,7 @@ import org.apache.linkis.ecm.server.conf.ECMConfiguration import org.apache.linkis.ecm.server.conf.ECMConfiguration._ import org.apache.linkis.ecm.server.listener.{ECMClosedEvent, ECMReadyEvent} import org.apache.linkis.ecm.server.report.DefaultECMHealthReport -import org.apache.linkis.ecm.server.service.{ECMHealthService, EngineConnListService} +import org.apache.linkis.ecm.server.service.ECMHealthService import org.apache.linkis.ecm.server.util.ECMUtils import org.apache.linkis.manager.common.entity.enumeration.{NodeHealthy, NodeStatus} import org.apache.linkis.manager.common.entity.metrics.{NodeHealthyInfo, NodeOverLoadInfo} @@ -38,8 +38,6 @@ import org.apache.linkis.manager.common.protocol.node.{ import org.apache.linkis.rpc.Sender import org.apache.linkis.rpc.message.annotation.Receiver -import org.springframework.beans.factory.annotation.Autowired - import java.util.Date import java.util.concurrent.TimeUnit @@ -79,9 +77,6 @@ class DefaultECMHealthService extends ECMHealthService with ECMEventListener { TimeUnit.SECONDS ) - @Autowired - private var engineConnListService: EngineConnListService = _ - override def getLastEMHealthReport: ECMHealthReport = { val report = new DefaultECMHealthReport report.setNodeId(LinkisECMApplication.getECMServiceInstance.toString) @@ -89,7 +84,6 @@ class DefaultECMHealthService extends ECMHealthService with ECMEventListener { // todo report right metrics report.setTotalResource(maxResource) report.setProtectedResource(minResource) - report.setUsedResource(engineConnListService.getUsedResources) report.setReportTime(new Date().getTime) report.setRunningEngineConns( LinkisECMApplication.getContext.getECMMetrics.getRunningEngineConns @@ -117,7 +111,6 @@ class DefaultECMHealthService extends ECMHealthService with ECMEventListener { // todo report latest engineconn metrics resource.setMaxResource(maxResource) resource.setMinResource(minResource) - resource.setUsedResource(report.getUsedResource) heartbeat.setNodeResource(resource) heartbeat.setHeartBeatMsg("") val nodeHealthyInfo = new NodeHealthyInfo diff --git a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/service/impl/DefaultEngineConnKillService.java b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/service/impl/DefaultEngineConnKillService.java index cb8c7b1161..24f7797fba 100644 --- a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/service/impl/DefaultEngineConnKillService.java +++ b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/service/impl/DefaultEngineConnKillService.java @@ -20,17 +20,13 @@ import org.apache.commons.io.IOUtils; import org.apache.linkis.common.ServiceInstance; import org.apache.linkis.common.utils.Utils; -import org.apache.linkis.ecm.core.engineconn.EngineConn; import org.apache.linkis.ecm.server.conf.ECMConfiguration; import org.apache.linkis.ecm.server.service.EngineConnKillService; -import org.apache.linkis.ecm.server.service.EngineConnListService; import org.apache.linkis.engineconn.common.conf.EngineConnConf; import org.apache.linkis.governance.common.utils.GovernanceUtils; import org.apache.linkis.manager.common.protocol.engine.EngineStopRequest; import org.apache.linkis.manager.common.protocol.engine.EngineStopResponse; import org.apache.linkis.manager.common.protocol.engine.EngineSuicideRequest; -import org.apache.linkis.manager.label.entity.Label; -import org.apache.linkis.manager.label.entity.engine.EngineTypeLabel; import org.apache.linkis.rpc.message.annotation.Receiver; import org.apache.linkis.rpc.Sender; import org.apache.commons.lang3.StringUtils; @@ -44,7 +40,6 @@ import java.io.IOException; import java.util.ArrayList; import java.util.List; -import java.util.Optional; import java.util.concurrent.ThreadPoolExecutor; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -53,34 +48,37 @@ public class DefaultEngineConnKillService implements EngineConnKillService { private static final Logger logger = LoggerFactory.getLogger(DefaultEngineConnKillService.class); - private EngineConnListService engineConnListService; - - public void setEngineConnListService(EngineConnListService engineConnListService) { - this.engineConnListService = engineConnListService; - } - private static final ThreadPoolExecutor ecYarnAppKillService = Utils.newCachedThreadPool(10, "ECM-Kill-EC-Yarn-App", true); @Override @Receiver public EngineStopResponse dealEngineConnStop(EngineStopRequest engineStopRequest) { logger.info("received EngineStopRequest " + engineStopRequest); - EngineConn engineConn = getEngineConnByServiceInstance(engineStopRequest.getServiceInstance()); + String pid = null; + if("process".equals(engineStopRequest.getIdentifierType()) && StringUtils.isNotBlank(engineStopRequest.getIdentifier())){ + pid = engineStopRequest.getIdentifier(); + }else { + String processPort = engineStopRequest.getServiceInstance().getInstance().split(":")[1]; + pid = GovernanceUtils.findProcessIdentifier(processPort); + } + + logger.info("dealEngineConnStop return pid: {}", pid); EngineStopResponse response = new EngineStopResponse(); - if (null != engineConn) { - if(!killEngineConnByPid(engineConn)) { + if (StringUtils.isNotBlank(pid)) { + if(!killEngineConnByPid(pid, engineStopRequest.getServiceInstance())) { response.setStopStatus(false); - response.setMsg("Kill engine " + engineConn.getServiceInstance().toString() + " failed."); + response.setMsg("Kill engine " + engineStopRequest.getServiceInstance().toString() + " failed."); } else { response.setStopStatus(true); - response.setMsg("Kill engine " + engineConn.getServiceInstance().toString() + " succeed."); + response.setMsg("Kill engine " + engineStopRequest.getServiceInstance().toString() + " succeed."); } - killYarnAppIdOfOneEc(engineConn); + killYarnAppIdOfOneEc(engineStopRequest.getLogDirSuffix(), engineStopRequest.getServiceInstance(), + engineStopRequest.getEngineType()); } else { - logger.warn("Cannot find engineconn : " + engineStopRequest.getServiceInstance().toString() + " in this engineConnManager engineConn list, cannot kill."); - response.setStopStatus(true); - response.setMsg("EngineConn " + engineStopRequest.getServiceInstance().toString() + " was not found in this engineConnManager."); + logger.warn("Cannot find engineConn pid, try kill with rpc"); + response.setStopStatus(false); } + if (!response.getStopStatus()) { EngineSuicideRequest request = new EngineSuicideRequest(engineStopRequest.getServiceInstance(), engineStopRequest.getUser()); try { @@ -94,21 +92,21 @@ public EngineStopResponse dealEngineConnStop(EngineStopRequest engineStopRequest return response; } - public void killYarnAppIdOfOneEc(EngineConn engineConn) { - String engineConnInstance = engineConn.getServiceInstance().toString(); - logger.info("try to kill yarn app ids in the engine of ({}).", engineConnInstance); - String engineLogDir = engineConn.getEngineConnManagerEnv().engineConnLogDirs(); + public void killYarnAppIdOfOneEc(String logDirSuffix, ServiceInstance serviceInstance, String engineType) { + String engineConnInstance = serviceInstance.toString(); + String engineLogDir = ECMConfiguration.ENGINECONN_ROOT_DIR() + File.separator + logDirSuffix; + logger.info("try to kill yarn app ids in the engine of: [{}] engineLogDir: [{}]", engineConnInstance, engineLogDir); + final String errEngineLogPath = engineLogDir.concat(File.separator).concat("yarnApp"); - logger.info("try to parse the yarn app id from the engine err log file path: {}", errEngineLogPath); + logger.info("try to parse the yarn app id from the engine err log file path: [{}]", errEngineLogPath); File file = new File(errEngineLogPath); - if (file.exists()) - { + if (file.exists()) { ecYarnAppKillService.execute(() -> { BufferedReader in = null; try { in = new BufferedReader(new FileReader(errEngineLogPath)); String line; - String regex = getYarnAppRegexByEngineType(engineConn); + String regex = getYarnAppRegexByEngineType(engineType); if (StringUtils.isBlank(regex)) { return; } @@ -137,20 +135,10 @@ public void killYarnAppIdOfOneEc(EngineConn engineConn) { IOUtils.closeQuietly(in); } }); - } + } } - private String getYarnAppRegexByEngineType(EngineConn engineConn) { - List> labels = engineConn.getLabels(); - String engineType = ""; - if (labels != null && !labels.isEmpty()) { - Optional labelOptional = labels.stream().filter(label -> label instanceof EngineTypeLabel) - .map(label -> (EngineTypeLabel) label).findFirst(); - if (labelOptional.isPresent()) { - EngineTypeLabel engineTypeLabel = labelOptional.get(); - engineType = engineTypeLabel.getEngineType(); - } - } + private String getYarnAppRegexByEngineType(String engineType) { if (StringUtils.isBlank(engineType)) { return ""; } @@ -172,34 +160,17 @@ private String getYarnAppRegexByEngineType(EngineConn engineConn) { return regex; } - private EngineConn getEngineConnByServiceInstance(ServiceInstance serviceInstance) { - if (null == serviceInstance) { - return null; - } - List engineConnList = engineConnListService.getEngineConns(); - for (EngineConn engineConn : engineConnList) { - if (null != engineConn && serviceInstance.equals(engineConn.getServiceInstance())) { - return engineConn; - } - } - return null; - } - - private boolean killEngineConnByPid(EngineConn engineConn) { - logger.info("try to kill {} toString with pid({}).", engineConn.getServiceInstance().toString(), engineConn.getPid()); - if (StringUtils.isNotBlank(engineConn.getPid())) { + private boolean killEngineConnByPid(String processId, ServiceInstance serviceInstance) { + logger.info("try to kill {} toString with pid({}).", serviceInstance.toString(), processId); + if (StringUtils.isNotBlank(processId)) { if (ECMConfiguration.ECM_PROCESS_SCRIPT_KILL()) { - GovernanceUtils.killProcess(engineConn.getPid(), engineConn.getServiceInstance().toString(), true); - } else { - killProcessByKillCmd(engineConn.getPid(), engineConn.getServiceInstance().toString()); - } - if (isProcessAlive(engineConn.getPid())) { - return false; + GovernanceUtils.killProcess(processId, serviceInstance.toString(), true); } else { - return true; + killProcessByKillCmd(processId, serviceInstance.toString()); } + return !isProcessAlive(processId); } else { - logger.warn("cannot kill {} with empty pid.", engineConn.getServiceInstance().toString()); + logger.warn("cannot kill {} with empty pid.", serviceInstance.toString()); return false; } } diff --git a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/service/impl/DefaultEngineConnListService.scala b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/service/impl/DefaultEngineConnListService.scala deleted file mode 100644 index 4b9a59b4d7..0000000000 --- a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/service/impl/DefaultEngineConnListService.scala +++ /dev/null @@ -1,192 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.ecm.server.service.impl - -import org.apache.linkis.DataWorkCloudApplication -import org.apache.linkis.common.utils.{Logging, Utils} -import org.apache.linkis.ecm.core.engineconn.{EngineConn, YarnEngineConn} -import org.apache.linkis.ecm.core.launch.EngineConnLaunchRunner -import org.apache.linkis.ecm.core.listener.{ECMEvent, ECMEventListener} -import org.apache.linkis.ecm.server.LinkisECMApplication -import org.apache.linkis.ecm.server.converter.ECMEngineConverter -import org.apache.linkis.ecm.server.listener._ -import org.apache.linkis.ecm.server.service.EngineConnListService -import org.apache.linkis.manager.common.entity.enumeration.NodeStatus -import org.apache.linkis.manager.common.entity.resource.{Resource, ResourceType} -import org.apache.linkis.manager.common.protocol.engine.EngineStopRequest - -import org.apache.commons.lang3.StringUtils - -import java.util -import java.util.concurrent.ConcurrentHashMap - -import scala.collection.JavaConverters._ - -import com.google.common.collect.Interners - -class DefaultEngineConnListService - extends EngineConnListService - with ECMEventListener - with Logging { - - /** - * key:tickedId,value :engineConn - */ - private val engineConnMap = new ConcurrentHashMap[String, EngineConn] - - private var engineConnKillService: DefaultEngineConnKillService = _ - - val lock = Interners.newWeakInterner[String] - - override def init(): Unit = {} - - override def getEngineConn(engineConnId: String): Option[EngineConn] = Option( - engineConnMap.get(engineConnId) - ) - - override def getEngineConns: util.List[EngineConn] = - new util.ArrayList[EngineConn](engineConnMap.values()) - - override def addEngineConn(engineConn: EngineConn): Unit = { - logger.info(s"add engineConn ${engineConn.getServiceInstance} to engineConnMap") - if (LinkisECMApplication.isReady) { - engineConnMap.put(engineConn.getTickedId, engineConn) - } - } - - override def killEngineConn(engineConnId: String): Unit = { - var conn = engineConnMap.get(engineConnId) - if (conn != null) engineConnId.intern().synchronized { - conn = engineConnMap.get(engineConnId) - if (conn != null) { - Utils.tryAndWarn { - if (NodeStatus.Failed == conn.getStatus && StringUtils.isNotBlank(conn.getPid)) { - killECByEngineConnKillService(conn) - } else { - getEngineConnKillService().killYarnAppIdOfOneEc(conn) - } - conn.close() - } - engineConnMap.remove(engineConnId) - logger.info(s"engineconn ${conn.getServiceInstance} was closed.") - } - } - } - - override def getUsedResources: Resource = engineConnMap - .values() - .asScala - .map(_.getResource.getMinResource) - .fold(Resource.initResource(ResourceType.Default))(_ + _) - - override def submit(runner: EngineConnLaunchRunner): Option[EngineConn] = { - None - } - - def updateYarnAppId(event: YarnAppIdCallbackEvent): Unit = { - updateYarnEngineConn( - x => x.setApplicationId(event.protocol.applicationId), - event.protocol.nodeId - ) - } - - def updateYarnEngineConn(implicit - updateFunction: YarnEngineConn => Unit, - nodeId: String - ): Unit = { - lock.intern(nodeId) synchronized { - engineConnMap.get(nodeId) match { - case e: YarnEngineConn => updateFunction(e) - case e: EngineConn => - engineConnMap.put(nodeId, ECMEngineConverter.engineConn2YarnEngineConn(e)) - } - } - } - - def updateEngineConn(updateFunction: EngineConn => Unit, nodeId: String): Unit = { - lock.intern(nodeId) synchronized { - engineConnMap.get(nodeId) match { - case e: EngineConn => updateFunction(e) - case _ => - } - } - } - - def updateYarnInfo(event: YarnInfoCallbackEvent): Unit = { - updateYarnEngineConn(x => x.setApplicationURL(event.protocol.uri), event.protocol.nodeId) - } - - def updatePid(event: EngineConnPidCallbackEvent): Unit = { - updateEngineConn( - x => { - x.setPid(event.protocol.pid) - x.setServiceInstance(event.protocol.serviceInstance) - }, - event.protocol.ticketId - ) - } - - def updateEngineConnStatus(tickedId: String, updateStatus: NodeStatus): Unit = { - updateEngineConn(x => x.setStatus(updateStatus), tickedId) - if (NodeStatus.isCompleted(updateStatus)) { - logger.info(s" from engineConnMap to remove engineconn ticketId ${tickedId}") - killEngineConn(tickedId) - } - } - - override def onEvent(event: ECMEvent): Unit = { - logger.info(s"Deal event $event") - event match { - case event: ECMClosedEvent => shutdownEngineConns(event) - case event: YarnAppIdCallbackEvent => updateYarnAppId(event) - case event: YarnInfoCallbackEvent => updateYarnInfo(event) - case event: EngineConnPidCallbackEvent => updatePid(event) - case EngineConnAddEvent(engineConn) => addEngineConn(engineConn) - case EngineConnStatusChangeEvent(tickedId, updateStatus) => - updateEngineConnStatus(tickedId, updateStatus) - case _ => - } - } - - private def getEngineConnKillService(): DefaultEngineConnKillService = { - if (engineConnKillService == null) { - val applicationContext = DataWorkCloudApplication.getApplicationContext - engineConnKillService = applicationContext.getBean(classOf[DefaultEngineConnKillService]) - } - engineConnKillService - } - - private def shutdownEngineConns(event: ECMClosedEvent): Unit = { - logger.info("start to kill all engines belonging the ecm") - engineConnMap - .values() - .asScala - .foreach(engineconn => { - killECByEngineConnKillService(engineconn) - }) - logger.info("Done! success to kill all engines belonging the ecm") - } - - private def killECByEngineConnKillService(engineconn: EngineConn): Unit = { - logger.info(s"start to kill ec by engineConnKillService ${engineconn.getServiceInstance}") - val engineStopRequest = new EngineStopRequest() - engineStopRequest.setServiceInstance(engineconn.getServiceInstance) - getEngineConnKillService().dealEngineConnStop(engineStopRequest) - } - -} diff --git a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/service/impl/DefaultEngineConnPidCallbackService.scala b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/service/impl/DefaultEngineConnPidCallbackService.scala deleted file mode 100644 index 6fb2d4700d..0000000000 --- a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/service/impl/DefaultEngineConnPidCallbackService.scala +++ /dev/null @@ -1,39 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.ecm.server.service.impl - -import org.apache.linkis.common.utils.Logging -import org.apache.linkis.ecm.server.LinkisECMApplication -import org.apache.linkis.ecm.server.listener.EngineConnPidCallbackEvent -import org.apache.linkis.ecm.server.service.EngineConnPidCallbackService -import org.apache.linkis.governance.common.protocol.task.ResponseEngineConnPid -import org.apache.linkis.rpc.message.annotation.Receiver - -class DefaultEngineConnPidCallbackService extends EngineConnPidCallbackService with Logging { - - @Receiver - override def dealPid(protocol: ResponseEngineConnPid): Unit = { - // 1.设置pid - // 2.设置serviceInstance - // 3.状态为running - LinkisECMApplication.getContext.getECMSyncListenerBus.postToAll( - EngineConnPidCallbackEvent(protocol) - ) - } - -} diff --git a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/service/impl/DefaultEngineConnStatusCallbackService.scala b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/service/impl/DefaultEngineConnStatusCallbackService.scala deleted file mode 100644 index af627afece..0000000000 --- a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/service/impl/DefaultEngineConnStatusCallbackService.scala +++ /dev/null @@ -1,67 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.ecm.server.service.impl - -import org.apache.linkis.common.utils.Logging -import org.apache.linkis.ecm.server.LinkisECMApplication -import org.apache.linkis.ecm.server.conf.ECMConfiguration.MANAGER_SERVICE_NAME -import org.apache.linkis.ecm.server.listener.EngineConnStatusChangeEvent -import org.apache.linkis.ecm.server.service.EngineConnStatusCallbackService -import org.apache.linkis.manager.common.entity.enumeration.NodeStatus -import org.apache.linkis.manager.common.entity.enumeration.NodeStatus.{Failed, Running} -import org.apache.linkis.manager.common.protocol.engine.{ - EngineConnStatusCallback, - EngineConnStatusCallbackToAM -} -import org.apache.linkis.rpc.Sender -import org.apache.linkis.rpc.message.annotation.Receiver - -import org.springframework.stereotype.Service - -@Service -class DefaultEngineConnStatusCallbackService extends EngineConnStatusCallbackService with Logging { - - @Receiver - override def dealEngineConnStatusCallback(protocol: EngineConnStatusCallback): Unit = { - logger.info(s"Start to deal EngineConnStatusCallback $protocol") - - if (NodeStatus.isAvailable(protocol.status)) { - - LinkisECMApplication.getContext.getECMSyncListenerBus.postToAll( - EngineConnStatusChangeEvent(protocol.ticketId, Running) - ) - } else { - - Sender - .getSender(MANAGER_SERVICE_NAME) - .send( - EngineConnStatusCallbackToAM( - protocol.serviceInstance, - protocol.status, - protocol.initErrorMsg - ) - ) - LinkisECMApplication.getContext.getECMSyncListenerBus.postToAll( - EngineConnStatusChangeEvent(protocol.ticketId, Failed) - ) - } - - logger.info(s"Finished to deal EngineConnStatusCallback $protocol") - } - -} diff --git a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/service/impl/DefaultYarnCallbackService.scala b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/service/impl/DefaultYarnCallbackService.scala deleted file mode 100644 index 0bb2e1366f..0000000000 --- a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/service/impl/DefaultYarnCallbackService.scala +++ /dev/null @@ -1,40 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.ecm.server.service.impl - -import org.apache.linkis.ecm.server.LinkisECMApplication -import org.apache.linkis.ecm.server.listener.{YarnAppIdCallbackEvent, YarnInfoCallbackEvent} -import org.apache.linkis.ecm.server.service.YarnCallbackService -import org.apache.linkis.protocol.callback.{YarnAPPIdCallbackProtocol, YarnInfoCallbackProtocol} -import org.apache.linkis.rpc.message.annotation.Receiver - -class DefaultYarnCallbackService extends YarnCallbackService { - - @Receiver - override def dealApplicationId(protocol: YarnAPPIdCallbackProtocol): Unit = { - LinkisECMApplication.getContext.getECMSyncListenerBus.postToAll( - YarnAppIdCallbackEvent(protocol) - ) - } - - @Receiver - override def dealApplicationURI(protocol: YarnInfoCallbackProtocol): Unit = { - LinkisECMApplication.getContext.getECMSyncListenerBus.postToAll(YarnInfoCallbackEvent(protocol)) - } - -} diff --git a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/service/impl/ProcessEngineConnLaunchService.scala b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/service/impl/ProcessEngineConnLaunchService.scala index ad58ba723f..e6ab513f15 100644 --- a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/service/impl/ProcessEngineConnLaunchService.scala +++ b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/service/impl/ProcessEngineConnLaunchService.scala @@ -27,7 +27,7 @@ import org.apache.linkis.ecm.server.LinkisECMApplication import org.apache.linkis.ecm.server.conf.ECMConfiguration import org.apache.linkis.ecm.server.conf.ECMConfiguration.MANAGER_SERVICE_NAME import org.apache.linkis.ecm.server.exception.ECMErrorException -import org.apache.linkis.ecm.server.listener.EngineConnStatusChangeEvent +import org.apache.linkis.ecm.server.listener.EngineConnLaunchStatusChangeEvent import org.apache.linkis.ecm.server.service.LocalDirsHandleService import org.apache.linkis.manager.common.entity.enumeration.NodeStatus import org.apache.linkis.manager.common.entity.enumeration.NodeStatus._ @@ -128,11 +128,11 @@ abstract class ProcessEngineConnLaunchService extends AbstractEngineConnLaunchSe if (exitCode.exists(_ != 0)) { logger.info(s"engine ${tickedId} process exit ") LinkisECMApplication.getContext.getECMSyncListenerBus.postToAll( - EngineConnStatusChangeEvent(tickedId, ShuttingDown) + EngineConnLaunchStatusChangeEvent(tickedId, ShuttingDown) ) } else { LinkisECMApplication.getContext.getECMSyncListenerBus.postToAll( - EngineConnStatusChangeEvent(tickedId, Success) + EngineConnLaunchStatusChangeEvent(tickedId, Success) ) } } diff --git a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/spring/ECMSpringConfiguration.scala b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/spring/ECMSpringConfiguration.scala index 4a684bbec1..9084d829f5 100644 --- a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/spring/ECMSpringConfiguration.scala +++ b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/spring/ECMSpringConfiguration.scala @@ -39,12 +39,6 @@ class ECMSpringConfiguration { new DefaultECMContext } - @Bean - @ConditionalOnMissingBean - def getDefaultYarnCallbackService: YarnCallbackService = { - new DefaultYarnCallbackService - } - @Bean @ConditionalOnMissingBean def getBmlResourceLocalizationService( @@ -57,32 +51,12 @@ class ECMSpringConfiguration { service } - @Bean - @ConditionalOnMissingBean - def getDefaultLogCallbackService: LogCallbackService = { - null - } - @Bean @ConditionalOnMissingBean def getDefaultlocalDirsHandleService: LocalDirsHandleService = { new DefaultLocalDirsHandleService } - @Bean - @ConditionalOnMissingBean - def getDefaultEngineConnPidCallbackService: EngineConnPidCallbackService = { - new DefaultEngineConnPidCallbackService - } - - @Bean - @ConditionalOnMissingBean - def getDefaultEngineConnListService(context: ECMContext): EngineConnListService = { - implicit val service: DefaultEngineConnListService = new DefaultEngineConnListService - registerSyncListener(context) - service - } - @Bean @ConditionalOnMissingBean def getLinuxProcessEngineConnLaunchService( @@ -114,10 +88,8 @@ class ECMSpringConfiguration { @Bean @ConditionalOnMissingBean def getDefaultEngineConnKillService( - engineConnListService: EngineConnListService ): EngineConnKillService = { val service = new DefaultEngineConnKillService - service.setEngineConnListService(engineConnListService) service } diff --git a/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/test/scala/org/apache/linkis/engineconn/computation/executor/upstream/access/ECTaskEntranceInfoAccessHelper.scala b/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/test/scala/org/apache/linkis/engineconn/computation/executor/upstream/access/ECTaskEntranceInfoAccessHelper.scala deleted file mode 100644 index 42e79c52cc..0000000000 --- a/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/test/scala/org/apache/linkis/engineconn/computation/executor/upstream/access/ECTaskEntranceInfoAccessHelper.scala +++ /dev/null @@ -1,96 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.engineconn.computation.executor.upstream.access - -import org.apache.linkis.DataWorkCloudApplication -import org.apache.linkis.common.ServiceInstance -import org.apache.linkis.common.conf.{CommonVars, DWCArgumentsParser} -import org.apache.linkis.common.utils.Utils -import org.apache.linkis.engineconn.common.creation.DefaultEngineCreationContext -import org.apache.linkis.engineconn.core.util.EngineConnUtils -import org.apache.linkis.governance.common.conf.GovernanceCommonConf -import org.apache.linkis.governance.common.utils.EngineConnArgumentsParser -import org.apache.linkis.manager.engineplugin.common.launch.process.Environment -import org.apache.linkis.manager.label.builder.factory.{ - LabelBuilderFactory, - LabelBuilderFactoryContext -} -import org.apache.linkis.manager.label.entity.Label -import org.apache.linkis.server.conf.ServerConfiguration - -import org.apache.commons.lang3.StringUtils - -import java.util - -import org.slf4j.{Logger, LoggerFactory} - -object ECTaskEntranceInfoAccessHelper { - val logger: Logger = LoggerFactory.getLogger(ECTaskEntranceInfoAccessHelper.getClass) - - val engineCreationContext = new DefaultEngineCreationContext - val labelBuilderFactory: LabelBuilderFactory = LabelBuilderFactoryContext.getLabelBuilderFactory - - def initApp(args: Array[String]): Unit = { - val arguments = EngineConnArgumentsParser.getEngineConnArgumentsParser.parseToObj(args) - val engineConf = arguments.getEngineConnConfMap - engineCreationContext.setUser(engineConf.getOrElse("user", Utils.getJvmUser)) - engineCreationContext.setTicketId(engineConf.getOrElse("ticketId", "")) - val host = CommonVars(Environment.ECM_HOST.toString, "127.0.0.1").getValue - val port = CommonVars(Environment.ECM_PORT.toString, "80").getValue - engineCreationContext.setEMInstance( - ServiceInstance(GovernanceCommonConf.ENGINE_CONN_MANAGER_SPRING_NAME.getValue, s"$host:$port") - ) - val labels = new util.ArrayList[Label[_]] - val labelArgs = engineConf.filter(_._1.startsWith(EngineConnArgumentsParser.LABEL_PREFIX)) - if (labelArgs.nonEmpty) { - labelArgs.foreach { case (key, value) => - labels.add( - labelBuilderFactory - .createLabel[Label[_]](key.replace(EngineConnArgumentsParser.LABEL_PREFIX, ""), value) - ) - } - engineCreationContext.setLabels(labels) - } - val jMap = new java.util.HashMap[String, String](engineConf.size) - engineConf.foreach(kv => jMap.put(kv._1, kv._2)) - engineCreationContext.setOptions(jMap) - engineCreationContext.setArgs(args) - // EngineConnObject.setEngineCreationContext(engineCreationContext) - logger.info( - "Finished to init engineCreationContext: " + EngineConnUtils.GSON - .toJson(engineCreationContext) - ) - - logger.info("Spring is enabled, now try to start SpringBoot.") - logger.info("<--------------------Start SpringBoot App-------------------->") - val parser = DWCArgumentsParser.parse(engineCreationContext.getArgs) - DWCArgumentsParser.setDWCOptionMap(parser.getDWCConfMap) - val existsExcludePackages = ServerConfiguration.BDP_SERVER_EXCLUDE_PACKAGES.getValue - if (!StringUtils.isEmpty(existsExcludePackages)) { - DataWorkCloudApplication.setProperty( - ServerConfiguration.BDP_SERVER_EXCLUDE_PACKAGES.key, - existsExcludePackages - ) - } - // 加载spring类 - DataWorkCloudApplication.main(DWCArgumentsParser.formatSpringOptions(parser.getSpringConfMap)) - - logger.info("<--------------------SpringBoot App init succeed-------------------->") - } - -} diff --git a/linkis-computation-governance/linkis-engineconn/linkis-engineconn-core/src/main/scala/org/apache/linkis/engineconn/launch/EngineConnServer.scala b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-core/src/main/scala/org/apache/linkis/engineconn/launch/EngineConnServer.scala index 180798a772..14a0701d9d 100644 --- a/linkis-computation-governance/linkis-engineconn/linkis-engineconn-core/src/main/scala/org/apache/linkis/engineconn/launch/EngineConnServer.scala +++ b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-core/src/main/scala/org/apache/linkis/engineconn/launch/EngineConnServer.scala @@ -127,6 +127,7 @@ object EngineConnServer extends Logging { this.engineCreationContext.setEMInstance( ServiceInstance(GovernanceCommonConf.ENGINE_CONN_MANAGER_SPRING_NAME.getValue, s"$host:$port") ) + val labels = new ArrayBuffer[Label[_]] val labelArgs = engineConf.filter(_._1.startsWith(EngineConnArgumentsParser.LABEL_PREFIX)) if (labelArgs.nonEmpty) { diff --git a/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/org/apache/linkis/engineconn/callback/hook/CallbackEngineConnHook.scala b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/org/apache/linkis/engineconn/callback/hook/CallbackEngineConnHook.scala index 6c9e1e68f4..a5bf02f84f 100644 --- a/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/org/apache/linkis/engineconn/callback/hook/CallbackEngineConnHook.scala +++ b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/org/apache/linkis/engineconn/callback/hook/CallbackEngineConnHook.scala @@ -61,7 +61,7 @@ class CallbackEngineConnHook extends EngineConnHook with Logging { newMap.put("spring.mvc.servlet.path", ServerConfiguration.BDP_SERVER_RESTFUL_URI.getValue) DataWorkCloudApplication.main(DWCArgumentsParser.formatSpringOptions(newMap.toMap)) - val engineConnPidCallBack = new EngineConnPidCallback(engineCreationContext.getEMInstance) + val engineConnPidCallBack = new EngineConnPidCallback() Utils.tryAndError(engineConnPidCallBack.callback()) logger.info("<--------------------SpringBoot App init succeed-------------------->") } @@ -80,9 +80,7 @@ class CallbackEngineConnHook extends EngineConnHook with Logging { engineCreationContext: EngineCreationContext, throwable: Throwable ): Unit = { - val engineConnAfterStartCallback = new EngineConnAfterStartCallback( - engineCreationContext.getEMInstance - ) + val engineConnAfterStartCallback = new EngineConnAfterStartCallback val prefixMsg = Sender.getThisServiceInstance + s": log dir: ${EngineConnConf.getLogDir}," Utils.tryAndError( engineConnAfterStartCallback.callback( @@ -123,9 +121,7 @@ class CallbackEngineConnHook extends EngineConnHook with Logging { engineCreationContext: EngineCreationContext, engineConn: EngineConn ): Unit = { - val engineConnAfterStartCallback = new EngineConnAfterStartCallback( - engineCreationContext.getEMInstance - ) + val engineConnAfterStartCallback = new EngineConnAfterStartCallback Utils.tryAndError( engineConnAfterStartCallback.callback( EngineConnStatusCallback( diff --git a/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/org/apache/linkis/engineconn/callback/service/EngineConnAfterStartCallback.scala b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/org/apache/linkis/engineconn/callback/service/EngineConnAfterStartCallback.scala index fe6275ce67..d61e711f5d 100644 --- a/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/org/apache/linkis/engineconn/callback/service/EngineConnAfterStartCallback.scala +++ b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/org/apache/linkis/engineconn/callback/service/EngineConnAfterStartCallback.scala @@ -17,10 +17,7 @@ package org.apache.linkis.engineconn.callback.service -import org.apache.linkis.common.ServiceInstance - -class EngineConnAfterStartCallback(emInstance: ServiceInstance) - extends AbstractEngineConnStartUpCallback(emInstance) { +class EngineConnAfterStartCallback extends AbstractEngineConnStartUpCallback { override def callback(): Unit = {} } diff --git a/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/org/apache/linkis/engineconn/callback/service/EngineConnCallback.scala b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/org/apache/linkis/engineconn/callback/service/EngineConnCallback.scala index 8a028d0a90..1b629cf30c 100644 --- a/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/org/apache/linkis/engineconn/callback/service/EngineConnCallback.scala +++ b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/org/apache/linkis/engineconn/callback/service/EngineConnCallback.scala @@ -17,8 +17,8 @@ package org.apache.linkis.engineconn.callback.service -import org.apache.linkis.common.ServiceInstance import org.apache.linkis.common.utils.Logging +import org.apache.linkis.governance.common.conf.GovernanceCommonConf import org.apache.linkis.manager.common.entity.enumeration.NodeStatus import org.apache.linkis.manager.common.protocol.engine.EngineConnStatusCallback import org.apache.linkis.protocol.message.RequestProtocol @@ -26,31 +26,25 @@ import org.apache.linkis.rpc.Sender trait EngineConnCallback { - protected def getEMSender: Sender - def callback(): Unit } -abstract class AbstractEngineConnStartUpCallback(emInstance: ServiceInstance) - extends EngineConnCallback - with Logging { - - override protected def getEMSender: Sender = { - Sender.getSender(emInstance) - } +abstract class AbstractEngineConnStartUpCallback() extends EngineConnCallback with Logging { def callback(protocol: RequestProtocol): Unit = { protocol match { case protocol: EngineConnStatusCallback => if (protocol.status.equals(NodeStatus.Failed)) { - logger.error(s"protocol will send to em: ${protocol}") + logger.error(s"protocol will send to lm: ${protocol}") } else { - logger.info(s"protocol will send to em: ${protocol}") + logger.info(s"protocol will send to lm: ${protocol}") } case _ => } - getEMSender.send(protocol) + Sender + .getSender(GovernanceCommonConf.ENGINE_APPLICATION_MANAGER_SPRING_NAME.getValue) + .ask(protocol) } } diff --git a/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/org/apache/linkis/engineconn/callback/service/EngineConnPidCallback.scala b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/org/apache/linkis/engineconn/callback/service/EngineConnPidCallback.scala index 23a3f90a2b..f0995c0b99 100644 --- a/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/org/apache/linkis/engineconn/callback/service/EngineConnPidCallback.scala +++ b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/org/apache/linkis/engineconn/callback/service/EngineConnPidCallback.scala @@ -17,15 +17,13 @@ package org.apache.linkis.engineconn.callback.service -import org.apache.linkis.common.ServiceInstance import org.apache.linkis.engineconn.core.EngineConnObject import org.apache.linkis.governance.common.protocol.task.ResponseEngineConnPid import org.apache.linkis.rpc.Sender import java.lang.management.ManagementFactory -class EngineConnPidCallback(emInstance: ServiceInstance) - extends AbstractEngineConnStartUpCallback(emInstance) { +class EngineConnPidCallback extends AbstractEngineConnStartUpCallback { override def callback(): Unit = { val pid = ManagementFactory.getRuntimeMXBean.getName.split("@")(0) diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/LinkisManagerApplication.java b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/LinkisManagerApplication.java similarity index 96% rename from linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/LinkisManagerApplication.java rename to linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/LinkisManagerApplication.java index cfbefdbaaa..064d61a6fb 100644 --- a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/LinkisManagerApplication.java +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/LinkisManagerApplication.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.linkis.manager.am; +package org.apache.linkis.manager; import org.apache.linkis.LinkisBaseServerApp; diff --git a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/service/YarnCallbackService.scala b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/event/message/EngineConnPidCallbackEvent.java similarity index 62% rename from linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/service/YarnCallbackService.scala rename to linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/event/message/EngineConnPidCallbackEvent.java index 06a9c787c5..1202433fb2 100644 --- a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/service/YarnCallbackService.scala +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/event/message/EngineConnPidCallbackEvent.java @@ -15,14 +15,22 @@ * limitations under the License. */ -package org.apache.linkis.ecm.server.service +package org.apache.linkis.manager.am.event.message; -import org.apache.linkis.protocol.callback.{YarnAPPIdCallbackProtocol, YarnInfoCallbackProtocol} +import org.apache.linkis.governance.common.protocol.task.ResponseEngineConnPid; -trait YarnCallbackService { +import org.springframework.context.ApplicationEvent; - def dealApplicationId(protocol: YarnAPPIdCallbackProtocol): Unit +public class EngineConnPidCallbackEvent extends ApplicationEvent { - def dealApplicationURI(protocol: YarnInfoCallbackProtocol): Unit + private ResponseEngineConnPid protocol; + public EngineConnPidCallbackEvent(ResponseEngineConnPid protocol) { + super(protocol); + this.protocol = protocol; + } + + public ResponseEngineConnPid getProtocol() { + return protocol; + } } diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/restful/EMRestfulApi.java b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/restful/EMRestfulApi.java index 4477d60ef7..05cda79c57 100644 --- a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/restful/EMRestfulApi.java +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/restful/EMRestfulApi.java @@ -25,6 +25,7 @@ import org.apache.linkis.manager.am.exception.AMErrorCode; import org.apache.linkis.manager.am.exception.AMErrorException; import org.apache.linkis.manager.am.manager.EngineNodeManager; +import org.apache.linkis.manager.am.service.ECResourceInfoService; import org.apache.linkis.manager.am.service.em.ECMOperateService; import org.apache.linkis.manager.am.service.em.EMInfoService; import org.apache.linkis.manager.am.utils.AMUtils; @@ -33,6 +34,7 @@ import org.apache.linkis.manager.common.entity.metrics.NodeHealthyInfo; import org.apache.linkis.manager.common.entity.node.EMNode; import org.apache.linkis.manager.common.entity.node.EngineNode; +import org.apache.linkis.manager.common.entity.persistence.ECResourceInfoRecord; import org.apache.linkis.manager.common.protocol.OperateRequest$; import org.apache.linkis.manager.common.protocol.em.ECMOperateRequest; import org.apache.linkis.manager.common.protocol.em.ECMOperateRequest$; @@ -91,6 +93,7 @@ public class EMRestfulApi { @Autowired private ECMOperateService ecmOperateService; + @Autowired private ECResourceInfoService ecResourceInfoService; private LabelBuilderFactory stdLabelBuilderFactory = LabelBuilderFactoryContext.getLabelBuilderFactory(); @@ -323,7 +326,10 @@ public Message executeECMOperationByEC(HttpServletRequest req, @RequestBody Json return Message.error( "You have no permission to execute ECM Operation by this EngineConn " + serviceInstance); } - return executeECMOperation(engineNode.getEMNode(), new ECMOperateRequest(userName, parameters)); + return executeECMOperation( + engineNode.getEMNode(), + engineNode.getServiceInstance().getInstance(), + new ECMOperateRequest(userName, parameters)); } @ApiOperation( @@ -354,7 +360,7 @@ public Message executeECMOperation(HttpServletRequest req, @RequestBody JsonNode "Fail to process the operation parameters, cased by " + ExceptionUtils.getRootCauseMessage(e)); } - return executeECMOperation(ecmNode, new ECMOperateRequest(userName, parameters)); + return executeECMOperation(ecmNode, "", new ECMOperateRequest(userName, parameters)); } @ApiOperation(value = "openEngineLog", notes = "open Engine log", response = Message.class) @@ -377,9 +383,10 @@ public Message openEngineLog(HttpServletRequest req, @RequestBody JsonNode jsonN String userName = ModuleUserUtils.getOperationUser(req, "openEngineLog"); EMNode ecmNode; Map parameters; + String engineInstance; try { String emInstance = jsonNode.get("emInstance").asText(); - String engineInstance = jsonNode.get("instance").asText(); + engineInstance = jsonNode.get("instance").asText(); ServiceInstance serviceInstance = EngineRestfulApi.getServiceInstance(jsonNode); logger.info("User {} try to open engine: {} log.", userName, serviceInstance); ecmNode = @@ -416,10 +423,12 @@ public Message openEngineLog(HttpServletRequest req, @RequestBody JsonNode jsonN logger.error("Failed to open engine log, error:", e); return Message.error(e.getMessage()); } - return executeECMOperation(ecmNode, new ECMOperateRequest(userName, parameters)); + return executeECMOperation( + ecmNode, engineInstance, new ECMOperateRequest(userName, parameters)); } - private Message executeECMOperation(EMNode ecmNode, ECMOperateRequest ecmOperateRequest) { + private Message executeECMOperation( + EMNode ecmNode, String engineInstance, ECMOperateRequest ecmOperateRequest) { String operationName = OperateRequest$.MODULE$.getOperationName(ecmOperateRequest.parameters()); if (ArrayUtils.contains(adminOperations, operationName) && Configuration.isNotAdmin(ecmOperateRequest.user())) { @@ -434,6 +443,18 @@ private Message executeECMOperation(EMNode ecmNode, ECMOperateRequest ecmOperate + " admin Operation in ECM " + ecmNode.getServiceInstance()); } + + // fill in logDirSuffix + if (StringUtils.isNotBlank(engineInstance) + && Objects.isNull(ecmOperateRequest.parameters().get("logDirSuffix"))) { + ECResourceInfoRecord ecResourceInfoRecord = + ecResourceInfoService.getECResourceInfoRecordByInstance(engineInstance); + if (Objects.isNull(ecResourceInfoRecord)) { + return Message.error("ECM instance: " + ecmNode.getServiceInstance() + " not exist "); + } + ecmOperateRequest.parameters().put("logDirSuffix", ecResourceInfoRecord.getLogDirSuffix()); + } + ECMOperateResponse engineOperateResponse = ecmOperateService.executeOperation(ecmNode, ecmOperateRequest); diff --git a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/service/EngineConnPidCallbackService.scala b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/service/EngineConnPidCallbackService.java similarity index 82% rename from linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/service/EngineConnPidCallbackService.scala rename to linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/service/EngineConnPidCallbackService.java index 8e83394738..194eea590d 100644 --- a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/service/EngineConnPidCallbackService.scala +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/service/EngineConnPidCallbackService.java @@ -15,12 +15,11 @@ * limitations under the License. */ -package org.apache.linkis.ecm.server.service +package org.apache.linkis.manager.am.service; -import org.apache.linkis.governance.common.protocol.task.ResponseEngineConnPid - -trait EngineConnPidCallbackService { - - def dealPid(protocol: ResponseEngineConnPid): Unit +import org.apache.linkis.governance.common.protocol.task.ResponseEngineConnPid; +@FunctionalInterface +public interface EngineConnPidCallbackService { + void dealPid(ResponseEngineConnPid protocol); } diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/service/engine/EngineConnStatusCallbackService.scala b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/service/EngineConnStatusCallbackService.java similarity index 79% rename from linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/service/engine/EngineConnStatusCallbackService.scala rename to linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/service/EngineConnStatusCallbackService.java index b4e498155c..477d49aa10 100644 --- a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/service/engine/EngineConnStatusCallbackService.scala +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/service/EngineConnStatusCallbackService.java @@ -15,12 +15,11 @@ * limitations under the License. */ -package org.apache.linkis.manager.am.service.engine +package org.apache.linkis.manager.am.service; -import org.apache.linkis.manager.common.protocol.engine.EngineConnStatusCallbackToAM +import org.apache.linkis.manager.common.protocol.engine.EngineConnStatusCallbackToAM; -trait EngineConnStatusCallbackService { - - def dealEngineConnStatusCallback(engineConnStatusCallbackToAM: EngineConnStatusCallbackToAM): Unit +public interface EngineConnStatusCallbackService { + void dealEngineConnStatusCallbackToAM(EngineConnStatusCallbackToAM engineConnStatusCallbackToAM); } diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/service/impl/DefaultEngineConnPidCallbackService.java b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/service/impl/DefaultEngineConnPidCallbackService.java new file mode 100644 index 0000000000..e9a8290b05 --- /dev/null +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/service/impl/DefaultEngineConnPidCallbackService.java @@ -0,0 +1,61 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.manager.am.service.impl; + +import org.apache.linkis.governance.common.protocol.task.ResponseEngineConnPid; +import org.apache.linkis.manager.am.manager.DefaultEngineNodeManager; +import org.apache.linkis.manager.am.service.EngineConnPidCallbackService; +import org.apache.linkis.manager.common.entity.node.EngineNode; +import org.apache.linkis.rpc.message.annotation.Receiver; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +@Service +public class DefaultEngineConnPidCallbackService implements EngineConnPidCallbackService { + private static final Logger logger = + LoggerFactory.getLogger(DefaultEngineConnPidCallbackService.class); + + @Autowired private DefaultEngineNodeManager defaultEngineNodeManager; + + @Receiver + @Override + public void dealPid(ResponseEngineConnPid protocol) { + // 设置pid + logger.info( + "DefaultEngineConnPidCallbackService dealPid serviceInstance: [{}] pid: [{}]" + + " ticketId: [{}]", + protocol.serviceInstance(), + protocol.pid(), + protocol.ticketId()); + + EngineNode engineNode = defaultEngineNodeManager.getEngineNode(protocol.serviceInstance()); + if (engineNode == null) { + logger.error( + "DefaultEngineConnPidCallbackService dealPid failed, engineNode is null, serviceInstance:{}", + protocol.serviceInstance()); + return; + } + + engineNode.setIdentifier(protocol.pid()); + defaultEngineNodeManager.updateEngine(engineNode); + } +} diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/service/impl/DefaultEngineConnStatusCallbackService.java b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/service/impl/DefaultEngineConnStatusCallbackService.java new file mode 100644 index 0000000000..12c45e2310 --- /dev/null +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/service/impl/DefaultEngineConnStatusCallbackService.java @@ -0,0 +1,146 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.manager.am.service.impl; + +import org.apache.linkis.manager.am.conf.AMConfiguration; +import org.apache.linkis.manager.am.service.EngineConnStatusCallbackService; +import org.apache.linkis.manager.am.service.engine.EngineStopService; +import org.apache.linkis.manager.common.constant.AMConstant; +import org.apache.linkis.manager.common.entity.enumeration.NodeStatus; +import org.apache.linkis.manager.common.entity.metrics.AMNodeMetrics; +import org.apache.linkis.manager.common.protocol.engine.EngineConnStatusCallback; +import org.apache.linkis.manager.common.protocol.engine.EngineConnStatusCallbackToAM; +import org.apache.linkis.manager.common.protocol.engine.EngineStopRequest; +import org.apache.linkis.manager.persistence.NodeMetricManagerPersistence; +import org.apache.linkis.manager.service.common.metrics.MetricsConverter; +import org.apache.linkis.rpc.Sender$; +import org.apache.linkis.rpc.message.annotation.Receiver; +import org.apache.linkis.server.BDPJettyServerHelper; + +import org.apache.commons.lang3.StringUtils; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +import java.io.UnsupportedEncodingException; +import java.util.HashMap; +import java.util.Locale; +import java.util.Map; + +import com.fasterxml.jackson.core.JsonProcessingException; + +@Service +public class DefaultEngineConnStatusCallbackService implements EngineConnStatusCallbackService { + private org.slf4j.Logger logger = + org.slf4j.LoggerFactory.getLogger(DefaultEngineConnStatusCallbackService.class); + + @Autowired private NodeMetricManagerPersistence nodeMetricManagerPersistence; + + @Autowired private MetricsConverter metricsConverter; + + @Autowired private EngineStopService engineStopService; + + private static final String[] canRetryLogs = + AMConfiguration.AM_CAN_RETRY_LOGS().getValue().split(";"); + + @Receiver + public void dealEngineConnStatusCallback(EngineConnStatusCallback protocol) { + logger.info( + "EngineConnStatusCallbackServiceImpl handle engineConnStatus callback serviceInstance: [{}] status: [{}]", + protocol.serviceInstance(), + protocol.status()); + if (!NodeStatus.isAvailable(protocol.status())) { + EngineStopRequest engineStopRequest = new EngineStopRequest(); + engineStopRequest.setServiceInstance(protocol.serviceInstance()); + engineStopRequest.setUser("hadoop"); + try { + engineStopService.stopEngine( + engineStopRequest, Sender$.MODULE$.getSender(Sender$.MODULE$.getThisServiceInstance())); + } catch (Exception e) { + logger.warn( + "DefaultEngineConnStatusCallbackService stopEngine failed, serviceInstance:{}", + engineStopRequest.getServiceInstance(), + e); + } + + dealEngineConnStatusCallbackToAM( + new EngineConnStatusCallbackToAM( + protocol.serviceInstance(), protocol.status(), protocol.initErrorMsg(), false)); + } + logger.info("Finished to deal EngineConnStatusCallback: {}", protocol); + } + + @Receiver + public void dealEngineConnStatusCallbackToAM( + EngineConnStatusCallbackToAM engineConnStatusCallbackToAM) { + if (engineConnStatusCallbackToAM.serviceInstance() == null) { + logger.warn("call back service instance is null"); + } + logger.info( + "EngineConnStatusCallbackServiceImpl start to deal engineConnStatusCallbackToAM {}", + engineConnStatusCallbackToAM); + + AMNodeMetrics nodeMetrics = new AMNodeMetrics(); + Map heartBeatMsg = new HashMap<>(); + int initErrorMsgMaxByteNum = 60000; + + String initErrorMsg = engineConnStatusCallbackToAM.initErrorMsg(); + try { + if (StringUtils.isNotBlank(initErrorMsg) + && initErrorMsg.getBytes("utf-8").length >= initErrorMsgMaxByteNum) { + initErrorMsg = initErrorMsg.substring(0, initErrorMsgMaxByteNum); + } + } catch (UnsupportedEncodingException e) { + logger.warn("dealEngineConnStatusCallbackToAM getBytes failed", e); + } + heartBeatMsg.put(AMConstant.START_REASON, initErrorMsg); + + if (engineConnStatusCallbackToAM.canRetry()) { + heartBeatMsg.put(AMConstant.EC_CAN_RETRY, engineConnStatusCallbackToAM.canRetry()); + } else if (matchRetryLog(engineConnStatusCallbackToAM.initErrorMsg())) { + logger.info("match canRetry log {}", engineConnStatusCallbackToAM.serviceInstance()); + heartBeatMsg.put(AMConstant.EC_CAN_RETRY, engineConnStatusCallbackToAM.canRetry()); + } + + try { + nodeMetrics.setHeartBeatMsg( + BDPJettyServerHelper.jacksonJson().writeValueAsString(heartBeatMsg)); + } catch (JsonProcessingException e) { + logger.warn("dealEngineConnStatusCallbackToAM writeValueAsString failed", e); + } + nodeMetrics.setServiceInstance(engineConnStatusCallbackToAM.serviceInstance()); + nodeMetrics.setStatus(metricsConverter.convertStatus(engineConnStatusCallbackToAM.status())); + + nodeMetricManagerPersistence.addOrupdateNodeMetrics(nodeMetrics); + logger.info("Finished to deal engineConnStatusCallbackToAM {}", engineConnStatusCallbackToAM); + } + + private boolean matchRetryLog(String errorMsg) { + boolean flag = false; + if (StringUtils.isNotBlank(errorMsg)) { + String errorMsgLowCase = errorMsg.toLowerCase(Locale.getDefault()); + for (String canRetry : canRetryLogs) { + if (errorMsgLowCase.contains(canRetry)) { + logger.info("match engineConn log fatal logs, is {}", canRetry); + flag = true; + } + } + } + return flag; + } +} diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/manager/DefaultEngineNodeManager.scala b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/manager/DefaultEngineNodeManager.scala index 86cf986d29..098d6e5c74 100644 --- a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/manager/DefaultEngineNodeManager.scala +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/manager/DefaultEngineNodeManager.scala @@ -25,7 +25,12 @@ import org.apache.linkis.manager.am.exception.{AMErrorCode, AMErrorException} import org.apache.linkis.manager.am.locker.EngineNodeLocker import org.apache.linkis.manager.common.constant.AMConstant import org.apache.linkis.manager.common.entity.enumeration.NodeStatus -import org.apache.linkis.manager.common.entity.node.{AMEngineNode, EngineNode, ScoreServiceInstance} +import org.apache.linkis.manager.common.entity.node.{ + AMEngineNode, + EngineNode, + RMNode, + ScoreServiceInstance +} import org.apache.linkis.manager.common.entity.persistence.PersistenceLabel import org.apache.linkis.manager.common.protocol.engine.{ EngineOperateRequest, @@ -105,7 +110,6 @@ class DefaultEngineNodeManager extends EngineNodeManager with Logging { val heartMsg = engine.getNodeHeartbeatMsg() engineNode.setNodeHealthyInfo(heartMsg.getHealthyInfo) engineNode.setNodeOverLoadInfo(heartMsg.getOverLoadInfo) - engineNode.setNodeResource(heartMsg.getNodeResource) engineNode.setNodeStatus(heartMsg.getStatus) engineNode } @@ -125,7 +129,9 @@ class DefaultEngineNodeManager extends EngineNodeManager with Logging { toState: NodeStatus ): Unit = {} - override def updateEngine(engineNode: EngineNode): Unit = {} + override def updateEngine(engineNode: EngineNode): Unit = { + nodeManagerPersistence.updateNodeInstance(engineNode) + } override def switchEngine(engineNode: EngineNode): EngineNode = { null diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/service/em/DefaultEMEngineService.scala b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/service/em/DefaultEMEngineService.scala index aba78a85ba..2ea0b8c5fb 100644 --- a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/service/em/DefaultEMEngineService.scala +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/service/em/DefaultEMEngineService.scala @@ -19,18 +19,31 @@ package org.apache.linkis.manager.am.service.em import org.apache.linkis.common.utils.Logging import org.apache.linkis.engineplugin.server.service.EngineConnLaunchService +import org.apache.linkis.governance.common.utils.ECPathUtils import org.apache.linkis.manager.am.exception.AMErrorException import org.apache.linkis.manager.am.manager.{EMNodeManager, EngineNodeManager} +import org.apache.linkis.manager.am.service.ECResourceInfoService import org.apache.linkis.manager.am.service.EMEngineService import org.apache.linkis.manager.common.constant.AMConstant import org.apache.linkis.manager.common.entity.node._ +import org.apache.linkis.manager.common.entity.persistence.{ + ECResourceInfoRecord, + PersistenceResource +} import org.apache.linkis.manager.common.protocol.em._ import org.apache.linkis.manager.common.protocol.engine.EngineStopRequest import org.apache.linkis.manager.common.utils.ManagerUtils import org.apache.linkis.manager.engineplugin.common.launch.entity.EngineConnBuildRequest import org.apache.linkis.manager.label.entity.{EngineNodeLabel, Label} import org.apache.linkis.manager.label.entity.em.EMInstanceLabel +import org.apache.linkis.manager.label.entity.engine.{ + EngineInstanceLabel, + EngineTypeLabel, + UserCreatorLabel +} import org.apache.linkis.manager.label.service.NodeLabelService +import org.apache.linkis.manager.rm.domain.RMLabelContainer +import org.apache.linkis.manager.rm.service.LabelResourceService import org.apache.linkis.manager.service.common.label.LabelFilter import org.apache.commons.collections.MapUtils @@ -57,9 +70,15 @@ class DefaultEMEngineService extends EMEngineService with Logging { @Autowired private var engineConnLaunchService: EngineConnLaunchService = _ + @Autowired + private var ecResourceInfoService: ECResourceInfoService = _ + @Autowired private var labelFilter: LabelFilter = _ + @Autowired + private var labelResourceService: LabelResourceService = _ + override def listEngines(getEMEnginesRequest: GetEMEnginesRequest): util.List[EngineNode] = { val emNode = new AMEMNode() emNode.setServiceInstance(getEMEnginesRequest.getEm) @@ -95,6 +114,50 @@ class DefaultEMEngineService extends EMEngineService with Logging { ) val engineStopRequest = new EngineStopRequest engineStopRequest.setServiceInstance(engineNode.getServiceInstance) + engineStopRequest.setIdentifierType(engineNode.getMark) + engineStopRequest.setIdentifier(engineNode.getIdentifier) + + val ecResourceInfo: ECResourceInfoRecord = + ecResourceInfoService.getECResourceInfoRecordByInstance( + engineNode.getServiceInstance.getInstance + ) + + if (ecResourceInfo != null) { + engineStopRequest.setEngineType(ecResourceInfo.getLabelValue.split(",")(1).split("-")(0)) + engineStopRequest.setLogDirSuffix(ecResourceInfo.getLogDirSuffix) + } else { + if (engineNode.getLabels.isEmpty) { + // node labels is empty, engine already been stopped + logger.info( + s"DefaultEMEngineService stopEngine node labels is empty, engine: ${engineStopRequest.getServiceInstance} have already been stopped." + ) + return + } + + val rMLabelContainer: RMLabelContainer = + labelResourceService.enrichLabels(engineNode.getLabels) + + val persistenceResource: PersistenceResource = + labelResourceService.getPersistenceResource(rMLabelContainer.getEngineInstanceLabel) + if (persistenceResource == null) { + // persistenceResource is null, engine already been stopped + logger.info( + s"DefaultEMEngineService stopEngine persistenceResource is null, engine: ${engineStopRequest.getServiceInstance} have already been stopped." + ) + return + } + + engineStopRequest.setEngineType(rMLabelContainer.getEngineTypeLabel.getEngineType) + engineStopRequest.setLogDirSuffix( + ECPathUtils + .getECLogDirSuffix( + rMLabelContainer.getEngineTypeLabel, + rMLabelContainer.getUserCreatorLabel, + persistenceResource.getTicketId + ) + ) + } + emNodeManager.stopEngine(engineStopRequest, emNode) // engineNodeManager.deleteEngineNode(engineNode) logger.info( diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/service/engine/DefaultEngineConnStatusCallbackService.scala b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/service/engine/DefaultEngineConnStatusCallbackService.scala deleted file mode 100644 index ccfcb7aa28..0000000000 --- a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/service/engine/DefaultEngineConnStatusCallbackService.scala +++ /dev/null @@ -1,104 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.manager.am.service.engine - -import org.apache.linkis.common.utils.Logging -import org.apache.linkis.manager.am.conf.AMConfiguration -import org.apache.linkis.manager.common.constant.AMConstant -import org.apache.linkis.manager.common.entity.metrics.AMNodeMetrics -import org.apache.linkis.manager.common.protocol.engine.EngineConnStatusCallbackToAM -import org.apache.linkis.manager.persistence.NodeMetricManagerPersistence -import org.apache.linkis.manager.service.common.metrics.MetricsConverter -import org.apache.linkis.rpc.message.annotation.Receiver -import org.apache.linkis.server.BDPJettyServerHelper - -import org.apache.commons.lang3.StringUtils - -import org.springframework.beans.factory.annotation.Autowired -import org.springframework.stereotype.Service - -import java.util -import java.util.Locale - -@Service -class DefaultEngineConnStatusCallbackService extends EngineConnStatusCallbackService with Logging { - - @Autowired - private var nodeMetricManagerPersistence: NodeMetricManagerPersistence = _ - - @Autowired - private var metricsConverter: MetricsConverter = _ - - private val canRetryLogs = AMConfiguration.AM_CAN_RETRY_LOGS.getValue.split(";") - - // The heartBeatMsg field is of type text, mysql text max byte num is 65535 - private val initErrorMsgMaxByteNum = 60000 - - @Receiver - override def dealEngineConnStatusCallback( - engineConnStatusCallbackToAM: EngineConnStatusCallbackToAM - ): Unit = { - - if (null == engineConnStatusCallbackToAM.serviceInstance) { - logger.warn(s"call back service instance is null") - } - logger.info(s"Start to deal engineConnStatusCallbackToAM $engineConnStatusCallbackToAM") - val nodeMetrics = new AMNodeMetrics - val heartBeatMsg: java.util.Map[String, Any] = new util.HashMap[String, Any]() - - var initErrorMsg = engineConnStatusCallbackToAM.initErrorMsg - if ( - StringUtils.isNotBlank(initErrorMsg) && initErrorMsg - .getBytes("utf-8") - .length >= initErrorMsgMaxByteNum - ) { - initErrorMsg = initErrorMsg.substring(0, initErrorMsgMaxByteNum) - } - heartBeatMsg.put(AMConstant.START_REASON, initErrorMsg) - - if (engineConnStatusCallbackToAM.canRetry) { - heartBeatMsg.put(AMConstant.EC_CAN_RETRY, engineConnStatusCallbackToAM.canRetry) - } else if (matchRetryLog(engineConnStatusCallbackToAM.initErrorMsg)) { - logger.info(s"match canRetry log ${engineConnStatusCallbackToAM.serviceInstance}") - heartBeatMsg.put(AMConstant.EC_CAN_RETRY, engineConnStatusCallbackToAM.canRetry) - } - - nodeMetrics.setHeartBeatMsg(BDPJettyServerHelper.jacksonJson.writeValueAsString(heartBeatMsg)) - nodeMetrics.setServiceInstance(engineConnStatusCallbackToAM.serviceInstance) - nodeMetrics.setStatus(metricsConverter.convertStatus(engineConnStatusCallbackToAM.status)) - - nodeMetricManagerPersistence.addOrupdateNodeMetrics(nodeMetrics) - logger.info(s"Finished to deal engineConnStatusCallbackToAM $engineConnStatusCallbackToAM") - - } - - private def matchRetryLog(errorMsg: String): Boolean = { - var flag = false - if (StringUtils.isNotBlank(errorMsg)) { - val errorMsgLowCase = errorMsg.toLowerCase(Locale.getDefault) - canRetryLogs.foreach(canRetry => - if (errorMsgLowCase.contains(canRetry)) { - logger.info(s"match engineConn log fatal logs,is $canRetry") - flag = true - } - ) - } - flag - } - -} diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/service/engine/DefaultEngineStopService.scala b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/service/engine/DefaultEngineStopService.scala index 801114c4bf..3267b698d7 100644 --- a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/service/engine/DefaultEngineStopService.scala +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/service/engine/DefaultEngineStopService.scala @@ -94,13 +94,17 @@ class DefaultEngineStopService extends AbstractEngineService with EngineStopServ logger.info(s" engineConn does not exist in db: $engineStopRequest ") return } + + val labels = nodeLabelService.getNodeLabels(engineStopRequest.getServiceInstance) + node.setLabels(labels) + // 1. request em to kill ec logger.info(s"Start to kill engine invoke enginePointer ${node.getServiceInstance}") Utils.tryAndErrorMsg { getEMService().stopEngine(node, node.getEMNode) logger.info(s"Finished to kill engine invoke enginePointer ${node.getServiceInstance}") }(s"Failed to stop engine ${node.getServiceInstance}") - node.setLabels(nodeLabelService.getNodeLabels(engineStopRequest.getServiceInstance)) + if (null == node.getNodeStatus) { node.setNodeStatus(NodeStatus.ShuttingDown) } diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/service/engine/EngineStopService.scala b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/service/engine/EngineStopService.scala index 228f3e0801..beca547906 100644 --- a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/service/engine/EngineStopService.scala +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/service/engine/EngineStopService.scala @@ -17,7 +17,6 @@ package org.apache.linkis.manager.am.service.engine -import org.apache.linkis.manager.am.vo.ResourceVo import org.apache.linkis.manager.common.entity.node.EngineNode import org.apache.linkis.manager.common.protocol.engine.{ EngineConnReleaseRequest, @@ -26,10 +25,6 @@ import org.apache.linkis.manager.common.protocol.engine.{ } import org.apache.linkis.rpc.Sender -import java.util - -import scala.collection.mutable - trait EngineStopService { def stopEngine(engineStopRequest: EngineStopRequest, sender: Sender): Unit diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/rm/restful/RMMonitorRest.scala b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/rm/restful/RMMonitorRest.scala index 8c59f1b17a..4356d16a96 100644 --- a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/rm/restful/RMMonitorRest.scala +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/rm/restful/RMMonitorRest.scala @@ -25,7 +25,6 @@ import org.apache.linkis.manager.common.entity.node.EngineNode import org.apache.linkis.manager.common.entity.resource._ import org.apache.linkis.manager.common.errorcode.ManagerCommonErrorCodeSummary._ import org.apache.linkis.manager.common.exception.RMErrorException -import org.apache.linkis.manager.common.serializer.NodeResourceSerializer import org.apache.linkis.manager.common.utils.ResourceUtils import org.apache.linkis.manager.label.builder.CombinedLabelBuilder import org.apache.linkis.manager.label.builder.factory.LabelBuilderFactoryContext @@ -54,7 +53,6 @@ import org.apache.linkis.server.{toScalaBuffer, BDPJettyServerHelper, Message} import org.apache.linkis.server.security.SecurityFilter import org.apache.linkis.server.utils.ModuleUserUtils -import org.apache.commons.collections4.ListUtils import org.apache.commons.lang3.StringUtils import org.springframework.beans.factory.annotation.Autowired @@ -71,19 +69,18 @@ import scala.collection.mutable import scala.collection.mutable.ArrayBuffer import com.fasterxml.jackson.databind.ObjectMapper +import com.fasterxml.jackson.module.scala.DefaultScalaModule import com.github.pagehelper.page.PageMethod import com.google.common.collect.Lists -import io.swagger.annotations.{Api, ApiImplicitParams, ApiModel, ApiOperation} -import org.json4s.DefaultFormats -import org.json4s.jackson.Serialization.write +import io.swagger.annotations.{Api, ApiOperation} @RestController @Api(tags = Array("resource management")) @RequestMapping(path = Array("/linkisManager/rm")) class RMMonitorRest extends Logging { - implicit val formats = DefaultFormats + ResourceSerializer + NodeResourceSerializer val mapper = new ObjectMapper() + mapper.registerModule(DefaultScalaModule) private val dateFormatLocal = new ThreadLocal[SimpleDateFormat]() { override protected def initialValue = new SimpleDateFormat("EEE MMM dd HH:mm:ss z yyyy") @@ -126,8 +123,11 @@ class RMMonitorRest extends Logging { var COMBINED_USERCREATOR_ENGINETYPE: String = _ - def appendMessageData(message: Message, key: String, value: AnyRef): Message = - message.data(key, mapper.readTree(write(value))) + def appendMessageData(message: Message, key: String, value: AnyRef): Message = { + val result = mapper.writeValueAsString(value) + logger.info(s"appendMessageData result: $result") + message.data(key, mapper.readTree(result)) + } @ApiOperation(value = "getApplicationList", notes = "get applicationList") @RequestMapping(path = Array("applicationlist"), method = Array(RequestMethod.POST)) @@ -143,92 +143,10 @@ class RMMonitorRest extends Logging { val engineType = if (param.get("engineType") == null) null else param.get("engineType").asInstanceOf[String] val nodes = getEngineNodes(userName, true) - val creatorToApplicationList = new mutable.HashMap[String, mutable.HashMap[String, Any]] - nodes.foreach { node => - val userCreatorLabel = node.getLabels.asScala - .find(_.isInstanceOf[UserCreatorLabel]) - .get - .asInstanceOf[UserCreatorLabel] - val engineTypeLabel = node.getLabels.asScala - .find(_.isInstanceOf[EngineTypeLabel]) - .get - .asInstanceOf[EngineTypeLabel] - if (getUserCreator(userCreatorLabel).equals(userCreator)) { - if (engineType == null || getEngineType(engineTypeLabel).equals(engineType)) { - if (!creatorToApplicationList.contains(userCreatorLabel.getCreator)) { - val applicationList = new mutable.HashMap[String, Any] - applicationList.put("engineInstances", new mutable.ArrayBuffer[Any]) - applicationList.put("usedResource", Resource.initResource(ResourceType.LoadInstance)) - applicationList.put("maxResource", Resource.initResource(ResourceType.LoadInstance)) - applicationList.put("minResource", Resource.initResource(ResourceType.LoadInstance)) - applicationList.put("lockedResource", Resource.initResource(ResourceType.LoadInstance)) - creatorToApplicationList.put(userCreatorLabel.getCreator, applicationList) - } - val applicationList = creatorToApplicationList(userCreatorLabel.getCreator) - applicationList.put( - "usedResource", - (if (applicationList("usedResource") == null) { - Resource.initResource(ResourceType.LoadInstance) - } else { - applicationList("usedResource") - .asInstanceOf[Resource] - }) + node.getNodeResource.getUsedResource - ) - applicationList.put( - "maxResource", - (if (applicationList("maxResource") == null) { - Resource.initResource(ResourceType.LoadInstance) - } else { - applicationList("maxResource") - .asInstanceOf[Resource] - }) + node.getNodeResource.getMaxResource - ) - applicationList.put( - "minResource", - (if (applicationList("minResource") == null) { - Resource.initResource(ResourceType.LoadInstance) - } else { - applicationList("minResource") - .asInstanceOf[Resource] - }) + node.getNodeResource.getMinResource - ) - applicationList.put( - "lockedResource", - (if (applicationList("lockedResource") == null) { - Resource.initResource(ResourceType.LoadInstance) - } else { - applicationList("lockedResource") - .asInstanceOf[Resource] - }) + node.getNodeResource.getLockedResource - ) - val engineInstance = new mutable.HashMap[String, Any] - engineInstance.put("creator", userCreatorLabel.getCreator) - engineInstance.put("engineType", engineTypeLabel.getEngineType) - engineInstance.put("instance", node.getServiceInstance.getInstance) - engineInstance.put("label", engineTypeLabel.getStringValue) - node.setNodeResource( - ResourceUtils.convertTo(node.getNodeResource, ResourceType.LoadInstance) - ) - engineInstance.put("resource", node.getNodeResource) - if (node.getNodeStatus == null) { - engineInstance.put("status", "Busy") - } else { - engineInstance.put("status", node.getNodeStatus.toString) - } - engineInstance.put("startTime", dateFormatLocal.get().format(node.getStartTime)) - engineInstance.put("owner", node.getOwner) - applicationList("engineInstances") - .asInstanceOf[mutable.ArrayBuffer[Any]] - .append(engineInstance) - } - } - } - val applications = creatorToApplicationList.map { creatorEntry => - val application = new mutable.HashMap[String, Any] - application.put("creator", creatorEntry._1) - application.put("applicationList", creatorEntry._2) - application - } + + val creatorToApplicationList = getCreatorToApplicationList(userCreator, engineType, nodes) + + val applications = getApplications(creatorToApplicationList) appendMessageData(message, "applications", applications) message } @@ -338,112 +256,12 @@ class RMMonitorRest extends Logging { node.getLabels.asScala.find(_.isInstanceOf[EngineTypeLabel]).get != null }) } + val userCreatorEngineTypeResourceMap = - new mutable.HashMap[String, mutable.HashMap[String, NodeResource]] - nodes.foreach { node => - val userCreatorLabel = node.getLabels.asScala - .find(_.isInstanceOf[UserCreatorLabel]) - .get - .asInstanceOf[UserCreatorLabel] - val engineTypeLabel = node.getLabels.asScala - .find(_.isInstanceOf[EngineTypeLabel]) - .get - .asInstanceOf[EngineTypeLabel] - val userCreator = getUserCreator(userCreatorLabel) - if (!userCreatorEngineTypeResourceMap.contains(userCreator)) { - userCreatorEngineTypeResourceMap.put(userCreator, new mutable.HashMap[String, NodeResource]) - } - val engineTypeResourceMap = userCreatorEngineTypeResourceMap.get(userCreator).get - val engineType = getEngineType(engineTypeLabel) - if (!engineTypeResourceMap.contains(engineType)) { - val nodeResource = CommonNodeResource.initNodeResource(ResourceType.LoadInstance) - engineTypeResourceMap.put(engineType, nodeResource) - } - val resource = engineTypeResourceMap.get(engineType).get - resource.setUsedResource(node.getNodeResource.getUsedResource + resource.getUsedResource) - // combined label - val combinedLabel = - combinedLabelBuilder.build("", Lists.newArrayList(userCreatorLabel, engineTypeLabel)); - var labelResource = labelResourceService.getLabelResource(combinedLabel) - if (labelResource == null) { - resource.setLeftResource(node.getNodeResource.getMaxResource - resource.getUsedResource) - } else { - labelResource = ResourceUtils.convertTo(labelResource, ResourceType.LoadInstance) - resource.setUsedResource(labelResource.getUsedResource) - resource.setLockedResource(labelResource.getLockedResource) - resource.setLeftResource(labelResource.getLeftResource) - resource.setMaxResource(labelResource.getMaxResource) - } - resource.getLeftResource match { - case dResource: DriverAndYarnResource => - resource.setLeftResource(dResource.loadInstanceResource) - case _ => - } - } - val userCreatorEngineTypeResources = userCreatorEngineTypeResourceMap.map { userCreatorEntry => - val userCreatorEngineTypeResource = new mutable.HashMap[String, Any] - userCreatorEngineTypeResource.put("userCreator", userCreatorEntry._1) - var totalUsedMemory: Long = 0L - var totalUsedCores: Int = 0 - var totalUsedInstances = 0 - var totalLockedMemory: Long = 0L - var totalLockedCores: Int = 0 - var totalLockedInstances: Int = 0 - var totalMaxMemory: Long = 0L - var totalMaxCores: Int = 0 - var totalMaxInstances: Int = 0 - val engineTypeResources = userCreatorEntry._2.map { engineTypeEntry => - val engineTypeResource = new mutable.HashMap[String, Any] - engineTypeResource.put("engineType", engineTypeEntry._1) - val engineResource = engineTypeEntry._2 - val usedResource = engineResource.getUsedResource.asInstanceOf[LoadInstanceResource] - val lockedResource = engineResource.getLockedResource.asInstanceOf[LoadInstanceResource] - val maxResource = engineResource.getMaxResource.asInstanceOf[LoadInstanceResource] - val usedMemory = usedResource.memory - val usedCores = usedResource.cores - val usedInstances = usedResource.instances - totalUsedMemory += usedMemory - totalUsedCores += usedCores - totalUsedInstances += usedInstances - val lockedMemory = lockedResource.memory - val lockedCores = lockedResource.cores - val lockedInstances = lockedResource.instances - totalLockedMemory += lockedMemory - totalLockedCores += lockedCores - totalLockedInstances += lockedInstances - val maxMemory = maxResource.memory - val maxCores = maxResource.cores - val maxInstances = maxResource.instances - totalMaxMemory += maxMemory - totalMaxCores += maxCores - totalMaxInstances += maxInstances + getUserCreatorEngineTypeResourceMap(nodes) + + val userCreatorEngineTypeResources = getUserResources(userCreatorEngineTypeResourceMap) - val memoryPercent = - if (maxMemory > 0) (usedMemory + lockedMemory) / maxMemory.toDouble else 0 - val coresPercent = - if (maxCores > 0) (usedCores + lockedCores) / maxCores.toDouble else 0 - val instancePercent = - if (maxInstances > 0) (usedInstances + lockedInstances) / maxInstances.toDouble else 0 - val maxPercent = Math.max(Math.max(memoryPercent, coresPercent), instancePercent) - engineTypeResource.put("percent", maxPercent.formatted("%.2f")) - engineTypeResource - } - val totalMemoryPercent = - if (totalMaxMemory > 0) (totalUsedMemory + totalLockedMemory) / totalMaxMemory.toDouble - else 0 - val totalCoresPercent = - if (totalMaxCores > 0) (totalUsedCores + totalLockedCores) / totalMaxCores.toDouble - else 0 - val totalInstancePercent = - if (totalMaxInstances > 0) { - (totalUsedInstances + totalLockedInstances) / totalMaxInstances.toDouble - } else 0 - val totalPercent = - Math.max(Math.max(totalMemoryPercent, totalCoresPercent), totalInstancePercent) - userCreatorEngineTypeResource.put("engineTypes", engineTypeResources) - userCreatorEngineTypeResource.put("percent", totalPercent.formatted("%.2f")) - userCreatorEngineTypeResource - } appendMessageData(message, "userResources", userCreatorEngineTypeResources) message } @@ -780,4 +598,245 @@ class RMMonitorRest extends Logging { .groupBy(_.getOwner) } + private def getUserResources( + userCreatorEngineTypeResourceMap: mutable.HashMap[ + String, + mutable.HashMap[String, NodeResource] + ] + ) = { + + val userCreatorEngineTypeResource = new util.HashMap[String, Any] + + val userResources = new util.ArrayList[Any]() + userCreatorEngineTypeResourceMap.map { userCreatorEntry => + userCreatorEngineTypeResource.put("userCreator", userCreatorEntry._1) + var totalUsedMemory: Long = 0L + var totalUsedCores: Int = 0 + var totalUsedInstances = 0 + var totalLockedMemory: Long = 0L + var totalLockedCores: Int = 0 + var totalLockedInstances: Int = 0 + var totalMaxMemory: Long = 0L + var totalMaxCores: Int = 0 + var totalMaxInstances: Int = 0 + + val engineTypeResources = new util.ArrayList[Any]() + for (engineTypeEntry <- userCreatorEntry._2) { + val engineTypeResource = new util.HashMap[String, Any] + engineTypeResource.put("engineType", engineTypeEntry._1) + val engineResource = engineTypeEntry._2 + val usedResource = engineResource.getUsedResource.asInstanceOf[LoadInstanceResource] + val lockedResource = engineResource.getLockedResource.asInstanceOf[LoadInstanceResource] + val maxResource = engineResource.getMaxResource.asInstanceOf[LoadInstanceResource] + val usedMemory = usedResource.memory + val usedCores = usedResource.cores + val usedInstances = usedResource.instances + totalUsedMemory += usedMemory + totalUsedCores += usedCores + totalUsedInstances += usedInstances + val lockedMemory = lockedResource.memory + val lockedCores = lockedResource.cores + val lockedInstances = lockedResource.instances + totalLockedMemory += lockedMemory + totalLockedCores += lockedCores + totalLockedInstances += lockedInstances + val maxMemory = maxResource.memory + val maxCores = maxResource.cores + val maxInstances = maxResource.instances + totalMaxMemory += maxMemory + totalMaxCores += maxCores + totalMaxInstances += maxInstances + + val memoryPercent = + if (maxMemory > 0) (usedMemory + lockedMemory) / maxMemory.toDouble else 0 + val coresPercent = + if (maxCores > 0) (usedCores + lockedCores) / maxCores.toDouble else 0 + val instancePercent = + if (maxInstances > 0) (usedInstances + lockedInstances) / maxInstances.toDouble else 0 + val maxPercent = Math.max(Math.max(memoryPercent, coresPercent), instancePercent) + engineTypeResource.put("percent", maxPercent.formatted("%.2f")) + engineTypeResources.add(engineTypeResource) + } + + val totalMemoryPercent = + if (totalMaxMemory > 0) (totalUsedMemory + totalLockedMemory) / totalMaxMemory.toDouble + else 0 + val totalCoresPercent = + if (totalMaxCores > 0) (totalUsedCores + totalLockedCores) / totalMaxCores.toDouble + else 0 + val totalInstancePercent = + if (totalMaxInstances > 0) { + (totalUsedInstances + totalLockedInstances) / totalMaxInstances.toDouble + } else 0 + val totalPercent = + Math.max(Math.max(totalMemoryPercent, totalCoresPercent), totalInstancePercent) + userCreatorEngineTypeResource.put("engineTypes", engineTypeResources) + userCreatorEngineTypeResource.put("percent", totalPercent.formatted("%.2f")) + userResources.add(userCreatorEngineTypeResource) + } + + userResources + } + + private def getUserCreatorEngineTypeResourceMap(nodes: Array[EngineNode]) = { + val userCreatorEngineTypeResourceMap = + new mutable.HashMap[String, mutable.HashMap[String, NodeResource]] + + for (node <- nodes) { + val userCreatorLabel = node.getLabels.asScala + .find(_.isInstanceOf[UserCreatorLabel]) + .get + .asInstanceOf[UserCreatorLabel] + val engineTypeLabel = node.getLabels.asScala + .find(_.isInstanceOf[EngineTypeLabel]) + .get + .asInstanceOf[EngineTypeLabel] + val userCreator = getUserCreator(userCreatorLabel) + + if (!userCreatorEngineTypeResourceMap.contains(userCreator)) { + userCreatorEngineTypeResourceMap.put(userCreator, new mutable.HashMap[String, NodeResource]) + } + val engineTypeResourceMap = userCreatorEngineTypeResourceMap.get(userCreator).get + val engineType = getEngineType(engineTypeLabel) + if (!engineTypeResourceMap.contains(engineType)) { + val nodeResource = CommonNodeResource.initNodeResource(ResourceType.LoadInstance) + engineTypeResourceMap.put(engineType, nodeResource) + } + val resource = engineTypeResourceMap.get(engineType).get + resource.setUsedResource(node.getNodeResource.getUsedResource + resource.getUsedResource) + // combined label + val combinedLabel = + combinedLabelBuilder.build("", Lists.newArrayList(userCreatorLabel, engineTypeLabel)); + var labelResource = labelResourceService.getLabelResource(combinedLabel) + if (labelResource == null) { + resource.setLeftResource(node.getNodeResource.getMaxResource - resource.getUsedResource) + } else { + labelResource = ResourceUtils.convertTo(labelResource, ResourceType.LoadInstance) + resource.setUsedResource(labelResource.getUsedResource) + resource.setLockedResource(labelResource.getLockedResource) + resource.setLeftResource(labelResource.getLeftResource) + resource.setMaxResource(labelResource.getMaxResource) + } + resource.getLeftResource match { + case dResource: DriverAndYarnResource => + resource.setLeftResource(dResource.loadInstanceResource) + case _ => + } + } + + userCreatorEngineTypeResourceMap + } + + private def getCreatorToApplicationList( + userCreator: String, + engineType: String, + nodes: Array[EngineNode] + ) = { + val creatorToApplicationList = new util.HashMap[String, util.HashMap[String, Any]] + nodes.foreach { node => + val userCreatorLabel = node.getLabels.asScala + .find(_.isInstanceOf[UserCreatorLabel]) + .get + .asInstanceOf[UserCreatorLabel] + val engineTypeLabel = node.getLabels.asScala + .find(_.isInstanceOf[EngineTypeLabel]) + .get + .asInstanceOf[EngineTypeLabel] + if (getUserCreator(userCreatorLabel).equals(userCreator)) { + if (engineType == null || getEngineType(engineTypeLabel).equals(engineType)) { + if (!creatorToApplicationList.containsKey(userCreatorLabel.getCreator)) { + val applicationList = new util.HashMap[String, Any] + applicationList.put("engineInstances", new util.ArrayList[Any]) + applicationList.put("usedResource", Resource.initResource(ResourceType.LoadInstance)) + applicationList.put("maxResource", Resource.initResource(ResourceType.LoadInstance)) + applicationList.put("minResource", Resource.initResource(ResourceType.LoadInstance)) + applicationList.put("lockedResource", Resource.initResource(ResourceType.LoadInstance)) + creatorToApplicationList.put(userCreatorLabel.getCreator, applicationList) + } + val applicationList = creatorToApplicationList.get(userCreatorLabel.getCreator) + applicationList.put( + "usedResource", + (if (applicationList.get("usedResource") == null) { + Resource.initResource(ResourceType.LoadInstance) + } else { + applicationList + .get("usedResource") + .asInstanceOf[Resource] + }) + node.getNodeResource.getUsedResource + ) + applicationList.put( + "maxResource", + (if (applicationList.get("maxResource") == null) { + Resource.initResource(ResourceType.LoadInstance) + } else { + applicationList + .get("maxResource") + .asInstanceOf[Resource] + }) + node.getNodeResource.getMaxResource + ) + applicationList.put( + "minResource", + (if (applicationList.get("minResource") == null) { + Resource.initResource(ResourceType.LoadInstance) + } else { + applicationList + .get("minResource") + .asInstanceOf[Resource] + }) + node.getNodeResource.getMinResource + ) + applicationList.put( + "lockedResource", + (if (applicationList.get("lockedResource") == null) { + Resource.initResource(ResourceType.LoadInstance) + } else { + applicationList + .get("lockedResource") + .asInstanceOf[Resource] + }) + node.getNodeResource.getLockedResource + ) + val engineInstance = new mutable.HashMap[String, Any] + engineInstance.put("creator", userCreatorLabel.getCreator) + engineInstance.put("engineType", engineTypeLabel.getEngineType) + engineInstance.put("instance", node.getServiceInstance.getInstance) + engineInstance.put("label", engineTypeLabel.getStringValue) + node.setNodeResource( + ResourceUtils.convertTo(node.getNodeResource, ResourceType.LoadInstance) + ) + engineInstance.put("resource", node.getNodeResource) + if (node.getNodeStatus == null) { + engineInstance.put("status", "Busy") + } else { + engineInstance.put("status", node.getNodeStatus.toString) + } + engineInstance.put( + "st" + + "artTime", + dateFormatLocal.get().format(node.getStartTime) + ) + engineInstance.put("owner", node.getOwner) + applicationList + .get("engineInstances") + .asInstanceOf[util.ArrayList[Any]] + .add(engineInstance) + } + } + } + creatorToApplicationList + } + + private def getApplications( + creatorToApplicationList: util.HashMap[String, util.HashMap[String, Any]] + ) = { + val applications = new util.ArrayList[util.HashMap[String, Any]]() + val iterator = creatorToApplicationList.entrySet().iterator(); + while (iterator.hasNext) { + val entry = iterator.next() + val application = new util.HashMap[String, Any] + application.put("creator", entry.getKey) + application.put("applicationList", entry.getValue) + applications.add(application) + } + applications + } + } diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/rm/service/impl/ResourceLogService.scala b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/rm/service/impl/ResourceLogService.scala index e332854bc2..2d67edb9a7 100644 --- a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/rm/service/impl/ResourceLogService.scala +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/rm/service/impl/ResourceLogService.scala @@ -25,7 +25,11 @@ import org.apache.linkis.manager.common.entity.resource.Resource import org.apache.linkis.manager.dao.ECResourceRecordMapper import org.apache.linkis.manager.label.entity.CombinedLabel import org.apache.linkis.manager.label.entity.em.EMInstanceLabel -import org.apache.linkis.manager.label.entity.engine.EngineInstanceLabel +import org.apache.linkis.manager.label.entity.engine.{ + EngineInstanceLabel, + EngineTypeLabel, + UserCreatorLabel +} import org.apache.linkis.manager.rm.domain.RMLabelContainer import org.apache.linkis.manager.rm.utils.RMUtils @@ -150,7 +154,11 @@ class ResourceLogService extends Logging { if (null == userCreatorEngineType) return var ecResourceInfoRecord = ecResourceRecordMapper.getECResourceInfoRecord(ticketId) if (ecResourceInfoRecord == null) { - val logDirSuffix = getECLogDirSuffix(labelContainer, ticketId) + val logDirSuffix = ECPathUtils.getECLogDirSuffix( + labelContainer.getEngineTypeLabel, + labelContainer.getUserCreatorLabel, + ticketId + ) val user = if (null != labelContainer.getUserCreatorLabel) labelContainer.getUserCreatorLabel.getUser else "" @@ -200,20 +208,6 @@ class ResourceLogService extends Logging { ecResourceRecordMapper.updateECResourceInfoRecord(ecResourceInfoRecord) } - def getECLogDirSuffix(labelContainer: RMLabelContainer, ticketId: String): String = { - val engineTypeLabel = labelContainer.getEngineTypeLabel - val userCreatorLabel = labelContainer.getUserCreatorLabel - if (null == engineTypeLabel || null == userCreatorLabel) { - return "" - } - val suffix = ECPathUtils.getECWOrkDirPathSuffix( - userCreatorLabel.getUser, - ticketId, - engineTypeLabel.getEngineType - ) - suffix + File.separator + "logs" - } - } object ChangeType { diff --git a/linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/java/org/apache/linkis/manager/common/entity/node/AMEMNode.java b/linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/java/org/apache/linkis/manager/common/entity/node/AMEMNode.java index 132bc32bbd..c234bc00ad 100644 --- a/linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/java/org/apache/linkis/manager/common/entity/node/AMEMNode.java +++ b/linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/java/org/apache/linkis/manager/common/entity/node/AMEMNode.java @@ -43,6 +43,7 @@ public class AMEMNode implements EMNode, ScoreServiceInstance { private String owner; private String mark; + private String identifier; private NodeTaskInfo nodeTaskInfo; @@ -139,6 +140,16 @@ public void setMark(String mark) { this.mark = mark; } + @Override + public String getIdentifier() { + return identifier; + } + + @Override + public void setIdentifier(String identifier) { + this.identifier = identifier; + } + @Override public NodeResource getNodeResource() { return nodeResource; diff --git a/linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/java/org/apache/linkis/manager/common/entity/node/AMEngineNode.java b/linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/java/org/apache/linkis/manager/common/entity/node/AMEngineNode.java index 40107aaba8..243927f133 100644 --- a/linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/java/org/apache/linkis/manager/common/entity/node/AMEngineNode.java +++ b/linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/java/org/apache/linkis/manager/common/entity/node/AMEngineNode.java @@ -48,6 +48,7 @@ public class AMEngineNode implements EngineNode, ScoreServiceInstance { private String owner; private String mark; + private String identifier; private NodeTaskInfo nodeTaskInfo; @@ -129,6 +130,16 @@ public void setMark(String mark) { this.mark = mark; } + @Override + public String getIdentifier() { + return identifier; + } + + @Override + public void setIdentifier(String identifier) { + this.identifier = identifier; + } + @Override public EMNode getEMNode() { return this.emNode; diff --git a/linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/java/org/apache/linkis/manager/common/entity/node/InfoRMNode.java b/linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/java/org/apache/linkis/manager/common/entity/node/InfoRMNode.java index c6a329e95c..660ff0cf4c 100644 --- a/linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/java/org/apache/linkis/manager/common/entity/node/InfoRMNode.java +++ b/linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/java/org/apache/linkis/manager/common/entity/node/InfoRMNode.java @@ -32,6 +32,7 @@ public class InfoRMNode implements RMNode { private String owner; private String mark; + private String identifier; private NodeStatus nodeStatus; @@ -79,6 +80,16 @@ public String getMark() { return mark; } + @Override + public String getIdentifier() { + return identifier; + } + + @Override + public void setIdentifier(String identifier) { + this.identifier = identifier; + } + @Override public Date getUpdateTime() { return updateTime; diff --git a/linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/java/org/apache/linkis/manager/common/entity/node/Node.java b/linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/java/org/apache/linkis/manager/common/entity/node/Node.java index 135ff76db5..1ff76a931f 100644 --- a/linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/java/org/apache/linkis/manager/common/entity/node/Node.java +++ b/linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/java/org/apache/linkis/manager/common/entity/node/Node.java @@ -44,4 +44,8 @@ public interface Node extends RequestProtocol { Date getStartTime(); void setStartTime(Date startTime); + + String getIdentifier(); + + void setIdentifier(String identifier); } diff --git a/linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/java/org/apache/linkis/manager/common/entity/persistence/PersistenceNode.java b/linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/java/org/apache/linkis/manager/common/entity/persistence/PersistenceNode.java index 006bcf06ff..7302ffd63a 100644 --- a/linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/java/org/apache/linkis/manager/common/entity/persistence/PersistenceNode.java +++ b/linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/java/org/apache/linkis/manager/common/entity/persistence/PersistenceNode.java @@ -26,6 +26,8 @@ public class PersistenceNode { private String owner; private String mark; + /** identifier if mark equals "process", then identifier equals pid */ + private String identifier; private Date updateTime; private Date createTime; @@ -40,6 +42,14 @@ public void setMark(String mark) { this.mark = mark; } + public String getIdentifier() { + return identifier; + } + + public void setIdentifier(String identifier) { + this.identifier = identifier; + } + public Integer getId() { return id; } diff --git a/linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/java/org/apache/linkis/manager/common/entity/persistence/PersistenceNodeEntity.java b/linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/java/org/apache/linkis/manager/common/entity/persistence/PersistenceNodeEntity.java index 3eefbc75cc..e119d5d82a 100644 --- a/linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/java/org/apache/linkis/manager/common/entity/persistence/PersistenceNodeEntity.java +++ b/linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/java/org/apache/linkis/manager/common/entity/persistence/PersistenceNodeEntity.java @@ -28,6 +28,7 @@ public class PersistenceNodeEntity implements Node { private ServiceInstance serviceInstance; private String owner; private String mark; + private String identifier; private NodeStatus nodeStatus; private Date startTime; @@ -88,6 +89,16 @@ public String getMark() { return this.mark; } + @Override + public String getIdentifier() { + return identifier; + } + + @Override + public void setIdentifier(String identifier) { + this.identifier = identifier; + } + public void setOwner(String owner) { this.owner = owner; } diff --git a/linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/java/org/apache/linkis/manager/common/entity/persistence/PersistenceNodeMetrics.java b/linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/java/org/apache/linkis/manager/common/entity/persistence/PersistenceNodeMetrics.java index 01ecc24fc9..12ddf17b28 100644 --- a/linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/java/org/apache/linkis/manager/common/entity/persistence/PersistenceNodeMetrics.java +++ b/linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/java/org/apache/linkis/manager/common/entity/persistence/PersistenceNodeMetrics.java @@ -21,6 +21,7 @@ import org.apache.linkis.manager.common.entity.metrics.NodeMetrics; import java.util.Date; +import java.util.Objects; public class PersistenceNodeMetrics implements NodeMetrics { @@ -59,7 +60,9 @@ public Integer getStatus() { } public void setStatus(Integer status) { - this.status = status; + if (Objects.nonNull(status)) { + this.status = status; + } } @Override diff --git a/linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/java/org/apache/linkis/manager/common/protocol/engine/EngineStopRequest.java b/linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/java/org/apache/linkis/manager/common/protocol/engine/EngineStopRequest.java index 90515c48bc..ce14d09c46 100644 --- a/linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/java/org/apache/linkis/manager/common/protocol/engine/EngineStopRequest.java +++ b/linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/java/org/apache/linkis/manager/common/protocol/engine/EngineStopRequest.java @@ -23,8 +23,13 @@ public class EngineStopRequest implements EngineRequest, RequestMethod { private ServiceInstance serviceInstance; - + private String logDirSuffix; + private String engineType; private String user; + /** identifierType, Reserved for ec containerized startup scenarios */ + private String identifierType; + /** identifier */ + private String identifier; public EngineStopRequest() {} @@ -41,6 +46,38 @@ public void setServiceInstance(ServiceInstance serviceInstance) { this.serviceInstance = serviceInstance; } + public String getLogDirSuffix() { + return logDirSuffix; + } + + public void setLogDirSuffix(String logDirSuffix) { + this.logDirSuffix = logDirSuffix; + } + + public String getEngineType() { + return engineType; + } + + public void setEngineType(String engineType) { + this.engineType = engineType; + } + + public String getIdentifierType() { + return identifierType; + } + + public void setIdentifierType(String identifierType) { + this.identifierType = identifierType; + } + + public String getIdentifier() { + return identifier; + } + + public void setIdentifier(String identifier) { + this.identifier = identifier; + } + public void setUser(String user) { this.user = user; } @@ -60,9 +97,21 @@ public String toString() { return "EngineStopRequest{" + "serviceInstance=" + serviceInstance + + ", logDirSuffix='" + + logDirSuffix + + '\'' + + ", engineType='" + + engineType + + '\'' + ", user='" + user + '\'' + + ", identifierType='" + + identifierType + + '\'' + + ", identifier='" + + identifier + + '\'' + '}'; } } diff --git a/linkis-computation-governance/linkis-manager/linkis-manager-persistence/src/main/java/org/apache/linkis/manager/dao/NodeManagerMapper.java b/linkis-computation-governance/linkis-manager/linkis-manager-persistence/src/main/java/org/apache/linkis/manager/dao/NodeManagerMapper.java index 44dd771b4c..8c5f6b99ae 100644 --- a/linkis-computation-governance/linkis-manager/linkis-manager-persistence/src/main/java/org/apache/linkis/manager/dao/NodeManagerMapper.java +++ b/linkis-computation-governance/linkis-manager/linkis-manager-persistence/src/main/java/org/apache/linkis/manager/dao/NodeManagerMapper.java @@ -41,7 +41,7 @@ void updateNodeInstance( List getAllNodes(); - void updateNodeInstanceOverload(@Param("persistenceNode") PersistenceNode persistenceNode); + void updateNodeInstanceByInstance(@Param("persistenceNode") PersistenceNode persistenceNode); Integer getNodeInstanceId(@Param("instance") String instance); diff --git a/linkis-computation-governance/linkis-manager/linkis-manager-persistence/src/main/java/org/apache/linkis/manager/persistence/NodeMetricManagerPersistence.java b/linkis-computation-governance/linkis-manager/linkis-manager-persistence/src/main/java/org/apache/linkis/manager/persistence/NodeMetricManagerPersistence.java index afbf48b9f7..1a0887df06 100644 --- a/linkis-computation-governance/linkis-manager/linkis-manager-persistence/src/main/java/org/apache/linkis/manager/persistence/NodeMetricManagerPersistence.java +++ b/linkis-computation-governance/linkis-manager/linkis-manager-persistence/src/main/java/org/apache/linkis/manager/persistence/NodeMetricManagerPersistence.java @@ -39,7 +39,7 @@ public interface NodeMetricManagerPersistence { * @param nodeMetrics * @throws PersistenceErrorException */ - void addOrupdateNodeMetrics(NodeMetrics nodeMetrics) throws PersistenceErrorException; + void addOrupdateNodeMetrics(NodeMetrics nodeMetrics); /** * 获取多个节点的 metrics列表 diff --git a/linkis-computation-governance/linkis-manager/linkis-manager-persistence/src/main/java/org/apache/linkis/manager/persistence/impl/DefaultNodeManagerPersistence.java b/linkis-computation-governance/linkis-manager/linkis-manager-persistence/src/main/java/org/apache/linkis/manager/persistence/impl/DefaultNodeManagerPersistence.java index 86f202acee..4c6df1e16c 100644 --- a/linkis-computation-governance/linkis-manager/linkis-manager-persistence/src/main/java/org/apache/linkis/manager/persistence/impl/DefaultNodeManagerPersistence.java +++ b/linkis-computation-governance/linkis-manager/linkis-manager-persistence/src/main/java/org/apache/linkis/manager/persistence/impl/DefaultNodeManagerPersistence.java @@ -40,6 +40,7 @@ import java.util.ArrayList; import java.util.Date; import java.util.List; +import java.util.Objects; import java.util.stream.Collectors; import com.google.common.collect.Lists; @@ -183,24 +184,23 @@ public List getAllNodes() throws PersistenceErrorException { } @Override - public void updateNodeInstance(Node node) throws PersistenceErrorException { + public void updateNodeInstance(Node node) { - if (null != node) { + if (Objects.nonNull(node)) { PersistenceNode persistenceNode = new PersistenceNode(); persistenceNode.setInstance(node.getServiceInstance().getInstance()); persistenceNode.setName(node.getServiceInstance().getApplicationName()); - persistenceNode.setOwner(node.getOwner()); persistenceNode.setMark(node.getMark()); - persistenceNode.setCreateTime(new Date()); persistenceNode.setUpdateTime(new Date()); persistenceNode.setCreator(node.getOwner()); persistenceNode.setUpdator(node.getOwner()); - nodeManagerMapper.updateNodeInstanceOverload(persistenceNode); + persistenceNode.setIdentifier(node.getIdentifier()); + nodeManagerMapper.updateNodeInstanceByInstance(persistenceNode); } } @Override - public Node getNode(ServiceInstance serviceInstance) throws PersistenceErrorException { + public Node getNode(ServiceInstance serviceInstance) { String instance = serviceInstance.getInstance(); PersistenceNode nodeInstances = nodeManagerMapper.getNodeInstance(instance); if (null == nodeInstances) { diff --git a/linkis-computation-governance/linkis-manager/linkis-manager-persistence/src/main/java/org/apache/linkis/manager/persistence/impl/DefaultNodeMetricManagerPersistence.java b/linkis-computation-governance/linkis-manager/linkis-manager-persistence/src/main/java/org/apache/linkis/manager/persistence/impl/DefaultNodeMetricManagerPersistence.java index 60d6b7a590..c6134cd410 100644 --- a/linkis-computation-governance/linkis-manager/linkis-manager-persistence/src/main/java/org/apache/linkis/manager/persistence/impl/DefaultNodeMetricManagerPersistence.java +++ b/linkis-computation-governance/linkis-manager/linkis-manager-persistence/src/main/java/org/apache/linkis/manager/persistence/impl/DefaultNodeMetricManagerPersistence.java @@ -77,7 +77,7 @@ public void addNodeMetrics(NodeMetrics nodeMetrics) throws PersistenceErrorExcep } @Override - public void addOrupdateNodeMetrics(NodeMetrics nodeMetrics) throws PersistenceErrorException { + public void addOrupdateNodeMetrics(NodeMetrics nodeMetrics) { if (null == nodeMetrics.getServiceInstance()) { logger.warn( "The request of update node metrics was ignored, because the node metrics service instance is null"); diff --git a/linkis-computation-governance/linkis-manager/linkis-manager-persistence/src/main/resources/mapper/common/NodeManagerMapper.xml b/linkis-computation-governance/linkis-manager/linkis-manager-persistence/src/main/resources/mapper/common/NodeManagerMapper.xml index 09d923c007..ecfaddc1fe 100644 --- a/linkis-computation-governance/linkis-manager/linkis-manager-persistence/src/main/resources/mapper/common/NodeManagerMapper.xml +++ b/linkis-computation-governance/linkis-manager/linkis-manager-persistence/src/main/resources/mapper/common/NodeManagerMapper.xml @@ -20,73 +20,126 @@ - update linkis_cg_manager_engine_em set engine_instance = #{instance} where engine_instance = #{tickedId} + UPDATE linkis_cg_manager_engine_em + SET engine_instance = #{instance} + WHERE engine_instance = #{tickedId} - update linkis_cg_manager_label_service_instance set service_instance = #{instance} where service_instance = #{tickedId} + UPDATE linkis_cg_manager_label_service_instance + SET service_instance = #{instance} + WHERE service_instance = #{tickedId} - insert into linkis_cg_manager_service_instance(instance,name,owner,mark,update_time,create_time,updator,creator) - values(#{instance},#{name},#{owner},#{mark},#{updateTime},#{createTime},#{updator},#{creator}) + INSERT INTO linkis_cg_manager_service_instance (instance, name, owner, mark, update_time + , create_time, updator, creator) + VALUES (#{instance}, #{name}, #{owner}, #{mark}, #{updateTime} + , #{createTime}, #{updator}, #{creator}) - update linkis_cg_manager_service_instance set instance = #{persistenceNode.instance}, owner = #{persistenceNode.owner},mark = #{persistenceNode.mark},name = #{persistenceNode.name}, - update_time = #{persistenceNode.updateTime},updator = #{persistenceNode.updator},creator = #{persistenceNode.creator} where instance = #{instance} + UPDATE linkis_cg_manager_service_instance + SET instance = #{persistenceNode.instance}, owner = #{persistenceNode.owner}, mark = #{persistenceNode.mark}, name = #{persistenceNode.name}, update_time = #{persistenceNode.updateTime}, updator = #{persistenceNode.updator}, creator = #{persistenceNode.creator} + WHERE instance = #{instance} - delete from linkis_cg_manager_service_instance where instance = #{instance} + DELETE FROM linkis_cg_manager_service_instance + WHERE instance = #{instance} - update linkis_cg_manager_service_instance set owner = #{persistenceNode.owner},mark = #{persistenceNode.mark},name = #{persistenceNode.name}, - update_time = #{persistenceNode.updateTime},create_time = #{persistenceNode.createTime},updator = #{persistenceNode.updator} - ,creator = #{persistenceNode.creator} where instance = #{persistenceNode.instance} + UPDATE linkis_cg_manager_service_instance + SET owner = #{persistenceNode.owner}, mark = #{persistenceNode.mark}, name = #{persistenceNode.name}, update_time = #{persistenceNode.updateTime}, create_time = #{persistenceNode.createTime}, updator = #{persistenceNode.updator}, creator = #{persistenceNode.creator} + WHERE instance = #{persistenceNode.instance} + + + + UPDATE linkis_cg_manager_service_instance + SET + + mark = #{persistenceNode.mark}, + + + name = #{persistenceNode.name}, + + + update_time = #{persistenceNode.updateTime}, + + + updator = #{persistenceNode.updator}, + + + creator = #{persistenceNode.creator}, + + + identifier = #{persistenceNode.identifier} + + WHERE instance = #{persistenceNode.instance} - insert into linkis_cg_manager_engine_em (engine_instance, em_instance, update_time, create_time) - values(#{engineNodeInstance}, #{emNodeInstance}, now(), now()) + INSERT INTO linkis_cg_manager_engine_em (engine_instance, em_instance, update_time, create_time) + VALUES (#{engineNodeInstance}, #{emNodeInstance}, now(), now()) - delete from linkis_cg_manager_engine_em where engine_instance = #{engineNodeInstance} and em_instance = #{emNodeInstance} + DELETE FROM linkis_cg_manager_engine_em + WHERE engine_instance = #{engineNodeInstance} + AND em_instance = #{emNodeInstance} - SELECT t.*, metrics.instance_status, metrics.heartbeat_msg + SELECT t.*, metrics.instance_status FROM linkis_cg_manager_service_instance t, linkis_cg_manager_service_instance_metrics metrics WHERE t.name ="linkis-cg-engineconn" diff --git a/linkis-computation-governance/linkis-manager/linkis-manager-persistence/src/test/java/org/apache/linkis/manager/dao/NodeManagerMapperTest.java b/linkis-computation-governance/linkis-manager/linkis-manager-persistence/src/test/java/org/apache/linkis/manager/dao/NodeManagerMapperTest.java index 67b4bc354c..861b557fb3 100644 --- a/linkis-computation-governance/linkis-manager/linkis-manager-persistence/src/test/java/org/apache/linkis/manager/dao/NodeManagerMapperTest.java +++ b/linkis-computation-governance/linkis-manager/linkis-manager-persistence/src/test/java/org/apache/linkis/manager/dao/NodeManagerMapperTest.java @@ -93,7 +93,7 @@ void updateNodeInstanceOverload() { persistenceNode.setMark("testmark3"); persistenceNode.setUpdator("testupdator3"); persistenceNode.setCreator("testcreator3"); - nodeManagerMapper.updateNodeInstanceOverload(persistenceNode); + nodeManagerMapper.updateNodeInstanceByInstance(persistenceNode); PersistenceNode persistenceNodes = nodeManagerMapper.getNodeInstance("instance2"); assertTrue(persistenceNode.getName().equals(persistenceNodes.getName())); } diff --git a/linkis-dist/helm/charts/linkis/templates/configmap-init-sql.yaml b/linkis-dist/helm/charts/linkis/templates/configmap-init-sql.yaml index 175f2cb7ad..6d26ae863a 100644 --- a/linkis-dist/helm/charts/linkis/templates/configmap-init-sql.yaml +++ b/linkis-dist/helm/charts/linkis/templates/configmap-init-sql.yaml @@ -673,6 +673,7 @@ data: `name` varchar(32) COLLATE utf8_bin DEFAULT NULL, `owner` varchar(32) COLLATE utf8_bin DEFAULT NULL, `mark` varchar(32) COLLATE utf8_bin DEFAULT NULL, + `identifier` varchar(32) COLLATE utf8_bin DEFAULT NULL, `update_time` datetime DEFAULT CURRENT_TIMESTAMP, `create_time` datetime DEFAULT CURRENT_TIMESTAMP, `updator` varchar(32) COLLATE utf8_bin DEFAULT NULL, @@ -1183,12 +1184,12 @@ data: (select `relation`.`config_key_id` AS `config_key_id`, '' AS `config_value`, `relation`.`engine_type_label_id` AS `config_label_id` FROM linkis_ps_configuration_key_engine_relation relation INNER JOIN linkis_cg_manager_label label ON relation.engine_type_label_id = label.id AND label.label_value = '*-*,*-*'); - -- spark2.4.3 default configuration + -- spark default configuration insert into `linkis_ps_configuration_config_value` (`config_key_id`, `config_value`, `config_label_id`) (select `relation`.`config_key_id` AS `config_key_id`, '' AS `config_value`, `relation`.`engine_type_label_id` AS `config_label_id` FROM linkis_ps_configuration_key_engine_relation relation INNER JOIN linkis_cg_manager_label label ON relation.engine_type_label_id = label.id AND label.label_value = @SPARK_ALL); - -- hive1.2.1 default configuration + -- hive default configuration insert into `linkis_ps_configuration_config_value` (`config_key_id`, `config_value`, `config_label_id`) (select `relation`.`config_key_id` AS `config_key_id`, '' AS `config_value`, `relation`.`engine_type_label_id` AS `config_label_id` FROM linkis_ps_configuration_key_engine_relation relation INNER JOIN linkis_cg_manager_label label ON relation.engine_type_label_id = label.id AND label.label_value = @HIVE_ALL); diff --git a/linkis-dist/package/db/linkis_ddl.sql b/linkis-dist/package/db/linkis_ddl.sql index 94b94a8c32..371ace453a 100644 --- a/linkis-dist/package/db/linkis_ddl.sql +++ b/linkis-dist/package/db/linkis_ddl.sql @@ -680,6 +680,7 @@ CREATE TABLE `linkis_cg_manager_service_instance` ( `name` varchar(32) COLLATE utf8_bin DEFAULT NULL, `owner` varchar(32) COLLATE utf8_bin DEFAULT NULL, `mark` varchar(32) COLLATE utf8_bin DEFAULT NULL, + `identifier` varchar(32) COLLATE utf8_bin DEFAULT NULL, `update_time` datetime DEFAULT CURRENT_TIMESTAMP, `create_time` datetime DEFAULT CURRENT_TIMESTAMP, `updator` varchar(32) COLLATE utf8_bin DEFAULT NULL, diff --git a/linkis-dist/package/db/module/linkis_manager.sql b/linkis-dist/package/db/module/linkis_manager.sql index 1a404af9bb..c3128633e5 100644 --- a/linkis-dist/package/db/module/linkis_manager.sql +++ b/linkis-dist/package/db/module/linkis_manager.sql @@ -23,6 +23,7 @@ CREATE TABLE `linkis_cg_manager_service_instance` ( `name` varchar(32) COLLATE utf8_bin DEFAULT NULL, `owner` varchar(32) COLLATE utf8_bin DEFAULT NULL, `mark` varchar(32) COLLATE utf8_bin DEFAULT NULL, + `identifier` varchar(32) COLLATE utf8_bin DEFAULT NULL, `update_time` datetime DEFAULT CURRENT_TIMESTAMP, `create_time` datetime DEFAULT CURRENT_TIMESTAMP, `updator` varchar(32) COLLATE utf8_bin DEFAULT NULL, diff --git a/linkis-dist/package/sbin/ext/linkis-cg-linkismanager b/linkis-dist/package/sbin/ext/linkis-cg-linkismanager index 7a3849534a..9a3cd3317d 100644 --- a/linkis-dist/package/sbin/ext/linkis-cg-linkismanager +++ b/linkis-dist/package/sbin/ext/linkis-cg-linkismanager @@ -22,7 +22,7 @@ export SERVER_SUFFIX="linkis-computation-governance/linkis-cg-linkismanager" export SERVER_HEAP_SIZE="1G" -export SERVER_CLASS=org.apache.linkis.manager.am.LinkisManagerApplication +export SERVER_CLASS=org.apache.linkis.manager.LinkisManagerApplication if test -z "$MANAGER_HEAP_SIZE" then From 418b806f5ce7e95bb6d55d553f7427ef8446fc71 Mon Sep 17 00:00:00 2001 From: CharlieYan <16526279+charlieyan24@users.noreply.github.com> Date: Mon, 1 May 2023 00:06:15 +0800 Subject: [PATCH 125/261] fix: do not clean ec resource and service info when handler ecm close event(#4184) (#4510) --- .../rm/service/impl/DefaultResourceManager.scala | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/rm/service/impl/DefaultResourceManager.scala b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/rm/service/impl/DefaultResourceManager.scala index 6145f8bc0b..b8f3f5b6fa 100644 --- a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/rm/service/impl/DefaultResourceManager.scala +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/rm/service/impl/DefaultResourceManager.scala @@ -190,8 +190,8 @@ class DefaultResourceManager extends ResourceManager with Logging with Initializ } /** - * The registration method is mainly used to notify all RM nodes (including the node), and the - * instance is offline. 该注册方法，主要是用于通知所有的RM节点（包括本节点），下线该实例 + * The registration method is mainly used to notify all RM nodes , and the instance is offline. + * 该注册方法，主要是用于通知所有的RM节点（包括本节点），下线该实例 */ override def unregister(serviceInstance: ServiceInstance): Unit = { @@ -199,7 +199,6 @@ class DefaultResourceManager extends ResourceManager with Logging with Initializ LabelBuilderFactoryContext.getLabelBuilderFactory.createLabel(classOf[EMInstanceLabel]) eMInstanceLabel.setServiceName(serviceInstance.getApplicationName) eMInstanceLabel.setInstance(serviceInstance.getInstance) - val ecNodes = nodeManagerPersistence.getEngineNodeByEM(serviceInstance).asScala val lock = tryLockOneLabel(eMInstanceLabel, -1, Utils.getJvmUser) try { labelResourceService.removeResourceByLabel(eMInstanceLabel) @@ -217,13 +216,6 @@ class DefaultResourceManager extends ResourceManager with Logging with Initializ resourceLockService.unLock(lock) logger.info(s"Finished to clear ecm resource:${serviceInstance}") } - ecNodes.foreach { engineNode => - Utils.tryAndWarn { - engineNode.setLabels(nodeLabelService.getNodeLabels(engineNode.getServiceInstance)) - engineNode.setNodeStatus(NodeStatus.Failed) - engineStopService.engineConnInfoClear(engineNode) - } - } logger.info(s"Finished to clear ec for ecm ${serviceInstance}") } From e1bfa07b6cf981eb6447a26ef346b4202b03c659 Mon Sep 17 00:00:00 2001 From: casionone Date: Tue, 30 May 2023 14:25:42 +0800 Subject: [PATCH 126/261] update to 1.1.13-wedatasphere --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index a0e14f0fb4..39f20ade60 100644 --- a/pom.xml +++ b/pom.xml @@ -103,7 +103,7 @@ - 1.1.11-wedatasphere + 1.1.13-wedatasphere 2.9.2 2.4.3 2.7.2 From 0876f748ff66d55693a36fd038ce0b4d034384da Mon Sep 17 00:00:00 2001 From: peacewong Date: Tue, 30 May 2023 15:07:40 +0800 Subject: [PATCH 127/261] add sql connection --- .../ujes/jdbc/LinkisSQLConnection.scala | 465 ++++++++++++++++++ 1 file changed, 465 insertions(+) create mode 100644 linkis-computation-governance/linkis-jdbc-driver/src/main/scala/org/apache/linkis/ujes/jdbc/LinkisSQLConnection.scala diff --git a/linkis-computation-governance/linkis-jdbc-driver/src/main/scala/org/apache/linkis/ujes/jdbc/LinkisSQLConnection.scala b/linkis-computation-governance/linkis-jdbc-driver/src/main/scala/org/apache/linkis/ujes/jdbc/LinkisSQLConnection.scala new file mode 100644 index 0000000000..b800698766 --- /dev/null +++ b/linkis-computation-governance/linkis-jdbc-driver/src/main/scala/org/apache/linkis/ujes/jdbc/LinkisSQLConnection.scala @@ -0,0 +1,465 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.ujes.jdbc + +import org.apache.linkis.common.utils.{Logging, Utils} +import org.apache.linkis.manager.label.constant.LabelKeyConstant +import org.apache.linkis.manager.label.entity.engine.{EngineType, EngineTypeLabel, RunType} +import org.apache.linkis.manager.label.utils.EngineTypeLabelCreator +import org.apache.linkis.ujes.client.UJESClient +import org.apache.linkis.ujes.client.request.JobSubmitAction +import org.apache.linkis.ujes.client.response.JobExecuteResult +import org.apache.linkis.ujes.jdbc.UJESSQLDriverMain._ + +import org.apache.commons.lang3.StringUtils + +import java.{sql, util} +import java.sql.{ + Blob, + CallableStatement, + Clob, + Connection, + DatabaseMetaData, + NClob, + PreparedStatement, + ResultSet, + Savepoint, + SQLException, + SQLWarning, + SQLXML, + Statement, + Struct +} +import java.util.Properties +import java.util.concurrent.Executor + +import scala.collection.JavaConverters._ + +class LinkisSQLConnection(private[jdbc] val ujesClient: UJESClient, props: Properties) + extends Connection + with Logging { + + private[jdbc] var creator = "JDBCDriver" + + private[jdbc] var tableauFlag = false + + private[jdbc] val variableMap = { + val params = props.getProperty(PARAMS) + val map = new util.HashMap[String, AnyRef] + if (params != null) { + params.split(PARAM_SPLIT).map(_.split(KV_SPLIT)).foreach { + case Array(k, v) if k.startsWith(VARIABLE_HEADER) => + map.put(k.substring(VARIABLE_HEADER.length), v) + case Array(CREATOR, v) => + creator = v + case _ => + } + } + map + } + + def isTableau(): Boolean = { + val params = props.getProperty(PARAMS) + if (params != null) { + params.split(PARAM_SPLIT).map(_.split(KV_SPLIT)).foreach { + case Array(TABLEAU, v) => + tableauFlag = true + case _ => + } + } + tableauFlag + } + + private[jdbc] val dbName = + if (StringUtils.isNotBlank(props.getProperty(DB_NAME))) props.getProperty(DB_NAME) + else "default" + + private val runningSQLStatements = new util.LinkedList[Statement] + + private var closed = false + + private var inited = false + + private[jdbc] val user = props.getProperty(USER) + + private[jdbc] val serverURL = props.getProperty("URL") + + private val labelMap: util.Map[String, AnyRef] = new util.HashMap[String, AnyRef] + + private val startupParams: util.Map[String, AnyRef] = new util.HashMap[String, AnyRef] + + private val runtimeParams: util.Map[String, AnyRef] = new util.HashMap[String, AnyRef] + + private[jdbc] def getEngineType: EngineTypeLabel = { + val engineType: EngineTypeLabel = + EngineTypeLabelCreator.createEngineTypeLabel(EngineType.TRINO.toString) + if (props.containsKey(PARAMS)) { + val params = props.getProperty(PARAMS) + if (params != null & params.length() > 0) { + params.split(PARAM_SPLIT).map(_.split(KV_SPLIT)).foreach { + case Array(k, v) if k.equals(UJESSQLDriver.ENGINE_TYPE) => + return EngineTypeLabelCreator.createEngineTypeLabel(v) + case _ => + } + } + } + engineType + } + + private[jdbc] def throwWhenClosed[T](op: => T): T = + if (isClosed) throw new LinkisSQLException(LinkisSQLErrorCode.CONNECTION_CLOSED) + else op + + private def createStatementAndAdd[T <: Statement](op: => T): T = throwWhenClosed { + + val statement = op + runningSQLStatements.add(statement) + if (!inited) { + inited = true + Utils.tryAndWarn(statement.execute(s"USE $dbName")) + } + statement + } + + def getProps: Properties = props + + def removeStatement(statement: LinkisSQLStatement): Unit = runningSQLStatements.remove(statement) + + override def createStatement(): Statement = createStatementAndAdd(new LinkisSQLStatement(this)) + + override def prepareStatement(sql: String): LinkisSQLPreparedStatement = { + val statement = createStatementAndAdd(new LinkisSQLPreparedStatement(this, sql)) + statement.clearQuery() + statement + } + + override def createStatement(resultSetType: Int, resultSetConcurrency: Int): Statement = { + if (resultSetConcurrency != ResultSet.CONCUR_READ_ONLY) { + throw new SQLException( + "Statement with resultset concurrency " + resultSetConcurrency + " is not supported", + "HYC00" + ) + } + if (resultSetType == ResultSet.TYPE_SCROLL_SENSITIVE) { + throw new SQLException( + "Statement with resultset type " + resultSetType + " is not supported", + "HYC00" + ) + } + createStatementAndAdd(new LinkisSQLStatement(this)) + } + + override def prepareStatement(sql: String, autoGeneratedKeys: Int): PreparedStatement = + prepareStatement(sql) + + override def prepareStatement( + sql: String, + resultSetType: Int, + resultSetConcurrency: Int + ): PreparedStatement = prepareStatement(sql) + + override def getMetaData: DatabaseMetaData = throwWhenClosed(new UJESSQLDatabaseMetaData(this)) + + override def close(): Unit = { + runningSQLStatements.asScala.foreach { statement => Utils.tryQuietly(statement.close()) } + closed = true + } + + override def isClosed: Boolean = closed + + override def setReadOnly(readOnly: Boolean): Unit = + throw new LinkisSQLException( + LinkisSQLErrorCode.NOSUPPORT_CONNECTION, + "setReadOnly not supported" + ) + + override def isReadOnly: Boolean = false + + override def setCatalog(catalog: String): Unit = throwWhenClosed() + + override def getCatalog: String = "" + + override def setTransactionIsolation(level: Int): Unit = {} + + override def getTransactionIsolation: Int = Connection.TRANSACTION_NONE + + override def getWarnings: SQLWarning = null + + override def clearWarnings(): Unit = {} + + override def setAutoCommit(autoCommit: Boolean): Unit = {} + + override def getAutoCommit: Boolean = true + + override def commit(): Unit = {} + + override def prepareCall(sql: String): CallableStatement = + throw new LinkisSQLException( + LinkisSQLErrorCode.NOSUPPORT_CONNECTION, + "prepareCall not supported" + ) + + override def rollback(): Unit = + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_CONNECTION, "rollback not supported") + + override def nativeSQL(sql: String): String = + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_CONNECTION, "nativeSQL not supported") + + override def prepareCall( + sql: String, + resultSetType: Int, + resultSetConcurrency: Int + ): CallableStatement = + throw new LinkisSQLException( + LinkisSQLErrorCode.NOSUPPORT_CONNECTION, + "prepareCall not supported" + ) + + override def getTypeMap: util.Map[String, Class[_]] = + throw new LinkisSQLException( + LinkisSQLErrorCode.NOSUPPORT_CONNECTION, + "getTypeMap not supported" + ) + + override def setTypeMap(map: util.Map[String, Class[_]]): Unit = + throw new LinkisSQLException( + LinkisSQLErrorCode.NOSUPPORT_CONNECTION, + "setTypeMap not supported" + ) + + override def setHoldability(holdability: Int): Unit = throw new LinkisSQLException( + LinkisSQLErrorCode.NOSUPPORT_CONNECTION, + "setHoldability not supported" + ) + + override def getHoldability: Int = 0 + + override def setSavepoint(): Savepoint = + throw new LinkisSQLException( + LinkisSQLErrorCode.NOSUPPORT_CONNECTION, + "setSavepoint not supported" + ) + + override def setSavepoint(name: String): Savepoint = + throw new LinkisSQLException( + LinkisSQLErrorCode.NOSUPPORT_CONNECTION, + "setSavepoint not supported" + ) + + override def rollback(savepoint: Savepoint): Unit = + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_CONNECTION, "rollback not supported") + + override def releaseSavepoint(savepoint: Savepoint): Unit = throw new LinkisSQLException( + LinkisSQLErrorCode.NOSUPPORT_CONNECTION, + "releaseSavepoint not supported" + ) + + override def createStatement( + resultSetType: Int, + resultSetConcurrency: Int, + resultSetHoldability: Int + ): Statement = + throw new LinkisSQLException( + LinkisSQLErrorCode.NOSUPPORT_CONNECTION, + "createStatement not supported" + ) + + override def prepareStatement( + sql: String, + resultSetType: Int, + resultSetConcurrency: Int, + resultSetHoldability: Int + ): PreparedStatement = + throw new LinkisSQLException( + LinkisSQLErrorCode.NOSUPPORT_CONNECTION, + "prepareStatement not supported" + ) + + override def prepareCall( + sql: String, + resultSetType: Int, + resultSetConcurrency: Int, + resultSetHoldability: Int + ): CallableStatement = + throw new LinkisSQLException( + LinkisSQLErrorCode.NOSUPPORT_CONNECTION, + "prepareCall not supported" + ) + + override def prepareStatement(sql: String, columnIndexes: Array[Int]): PreparedStatement = + throw new LinkisSQLException( + LinkisSQLErrorCode.NOSUPPORT_CONNECTION, + "prepareStatement not supported" + ) + + override def prepareStatement(sql: String, columnNames: Array[String]): PreparedStatement = + throw new LinkisSQLException( + LinkisSQLErrorCode.NOSUPPORT_CONNECTION, + "prepareStatement not supported" + ) + + override def createClob(): Clob = + throw new LinkisSQLException( + LinkisSQLErrorCode.NOSUPPORT_CONNECTION, + "createClob not supported" + ) + + override def createBlob(): Blob = + throw new LinkisSQLException( + LinkisSQLErrorCode.NOSUPPORT_CONNECTION, + "createBlob not supported" + ) + + override def createNClob(): NClob = + throw new LinkisSQLException( + LinkisSQLErrorCode.NOSUPPORT_CONNECTION, + "createNClob not supported" + ) + + override def createSQLXML(): SQLXML = + throw new LinkisSQLException( + LinkisSQLErrorCode.NOSUPPORT_CONNECTION, + "createSQLXML not supported" + ) + + override def isValid(timeout: Int): Boolean = true + + override def setClientInfo(name: String, value: String): Unit = + throw new LinkisSQLException( + LinkisSQLErrorCode.NOSUPPORT_CONNECTION, + "setClientInfo not supported" + ) + + override def setClientInfo(properties: Properties): Unit = + throw new LinkisSQLException( + LinkisSQLErrorCode.NOSUPPORT_CONNECTION, + "properties not supported" + ) + + override def getClientInfo(name: String): String = + throw new LinkisSQLException( + LinkisSQLErrorCode.NOSUPPORT_CONNECTION, + "getClientInfo not supported" + ) + + override def getClientInfo: Properties = + throw new LinkisSQLException( + LinkisSQLErrorCode.NOSUPPORT_CONNECTION, + "getClientInfo not supported" + ) + + override def createArrayOf(typeName: String, elements: Array[AnyRef]): sql.Array = + throw new LinkisSQLException( + LinkisSQLErrorCode.NOSUPPORT_CONNECTION, + "createArrayOf not supported" + ) + + override def createStruct(typeName: String, attributes: Array[AnyRef]): Struct = + throw new LinkisSQLException( + LinkisSQLErrorCode.NOSUPPORT_CONNECTION, + "createStruct not supported" + ) + + override def setSchema(schema: String): Unit = throwWhenClosed { + if (StringUtils.isBlank(schema)) { + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_STATEMENT, "schema is empty!") + } + createStatement().execute("use " + schema) + } + + override def getSchema: String = throwWhenClosed { + val resultSet = createStatement().executeQuery("SELECT current_database()") + if (!resultSet.next()) { + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_STATEMENT, "Get schema failed!") + } + resultSet.getString(1) + } + + override def abort(executor: Executor): Unit = + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_CONNECTION, "abort not supported") + + override def setNetworkTimeout(executor: Executor, milliseconds: Int): Unit = + throw new LinkisSQLException( + LinkisSQLErrorCode.NOSUPPORT_CONNECTION, + "setNetworkTimeout not supported" + ) + + override def getNetworkTimeout: Int = throw new LinkisSQLException( + LinkisSQLErrorCode.NOSUPPORT_CONNECTION, + "getNetworkTimeout not supported" + ) + + override def unwrap[T](iface: Class[T]): T = + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_CONNECTION, "unwrap not supported") + + override def isWrapperFor(iface: Class[_]): Boolean = + throw new LinkisSQLException( + LinkisSQLErrorCode.NOSUPPORT_CONNECTION, + "isWrapperFor not supported" + ) + + def addLabels(labels: util.Map[String, AnyRef]): Unit = { + labelMap.putAll(labels) + } + + def addStartUpParams(params: util.Map[String, AnyRef]): Unit = { + startupParams.putAll(params) + } + + def addRuntimeParams(params: util.Map[String, AnyRef]): Unit = { + runtimeParams.putAll(params) + } + + def engineToCodeType(engine: String): String = { + val runType = EngineType.mapStringToEngineType(engine) match { + case EngineType.SPARK => RunType.SQL + case EngineType.HIVE => RunType.HIVE + case EngineType.TRINO => RunType.TRINO_SQL + case EngineType.PRESTO => RunType.PRESTO_SQL + case EngineType.ELASTICSEARCH => RunType.ES_SQL + case EngineType.JDBC => RunType.JDBC + case EngineType.PYTHON => RunType.SHELL + case _ => RunType.SQL + } + runType.toString + } + + private[jdbc] def toSubmit(code: String): JobExecuteResult = { + val engineTypeLabel = getEngineType + labelMap.put(LabelKeyConstant.ENGINE_TYPE_KEY, engineTypeLabel.getStringValue) + labelMap.put(LabelKeyConstant.USER_CREATOR_TYPE_KEY, s"$user-$creator") + labelMap.put(LabelKeyConstant.CODE_TYPE_KEY, engineToCodeType(engineTypeLabel.getEngineType)) + + val jobSubmitAction = JobSubmitAction.builder + .addExecuteCode(code) + .setStartupParams(startupParams) + .setUser(user) + .addExecuteUser(user) + .setLabels(labelMap) + .setRuntimeParams(runtimeParams) + .setVariableMap(variableMap) + .build + + val result = ujesClient.submit(jobSubmitAction) + if (result.getStatus != 0) { + throw new SQLException(result.getMessage) + } + result + } + +} From acb79ab2b3fc87c87945e8aebb431476c8059cb3 Mon Sep 17 00:00:00 2001 From: Casion Date: Tue, 30 May 2023 16:42:52 +0800 Subject: [PATCH 128/261] add sql connection (#168) Co-authored-by: peacewong --- .../ujes/jdbc/LinkisSQLConnection.scala | 465 ++++++++++++++++++ 1 file changed, 465 insertions(+) create mode 100644 linkis-computation-governance/linkis-jdbc-driver/src/main/scala/org/apache/linkis/ujes/jdbc/LinkisSQLConnection.scala diff --git a/linkis-computation-governance/linkis-jdbc-driver/src/main/scala/org/apache/linkis/ujes/jdbc/LinkisSQLConnection.scala b/linkis-computation-governance/linkis-jdbc-driver/src/main/scala/org/apache/linkis/ujes/jdbc/LinkisSQLConnection.scala new file mode 100644 index 0000000000..b800698766 --- /dev/null +++ b/linkis-computation-governance/linkis-jdbc-driver/src/main/scala/org/apache/linkis/ujes/jdbc/LinkisSQLConnection.scala @@ -0,0 +1,465 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.ujes.jdbc + +import org.apache.linkis.common.utils.{Logging, Utils} +import org.apache.linkis.manager.label.constant.LabelKeyConstant +import org.apache.linkis.manager.label.entity.engine.{EngineType, EngineTypeLabel, RunType} +import org.apache.linkis.manager.label.utils.EngineTypeLabelCreator +import org.apache.linkis.ujes.client.UJESClient +import org.apache.linkis.ujes.client.request.JobSubmitAction +import org.apache.linkis.ujes.client.response.JobExecuteResult +import org.apache.linkis.ujes.jdbc.UJESSQLDriverMain._ + +import org.apache.commons.lang3.StringUtils + +import java.{sql, util} +import java.sql.{ + Blob, + CallableStatement, + Clob, + Connection, + DatabaseMetaData, + NClob, + PreparedStatement, + ResultSet, + Savepoint, + SQLException, + SQLWarning, + SQLXML, + Statement, + Struct +} +import java.util.Properties +import java.util.concurrent.Executor + +import scala.collection.JavaConverters._ + +class LinkisSQLConnection(private[jdbc] val ujesClient: UJESClient, props: Properties) + extends Connection + with Logging { + + private[jdbc] var creator = "JDBCDriver" + + private[jdbc] var tableauFlag = false + + private[jdbc] val variableMap = { + val params = props.getProperty(PARAMS) + val map = new util.HashMap[String, AnyRef] + if (params != null) { + params.split(PARAM_SPLIT).map(_.split(KV_SPLIT)).foreach { + case Array(k, v) if k.startsWith(VARIABLE_HEADER) => + map.put(k.substring(VARIABLE_HEADER.length), v) + case Array(CREATOR, v) => + creator = v + case _ => + } + } + map + } + + def isTableau(): Boolean = { + val params = props.getProperty(PARAMS) + if (params != null) { + params.split(PARAM_SPLIT).map(_.split(KV_SPLIT)).foreach { + case Array(TABLEAU, v) => + tableauFlag = true + case _ => + } + } + tableauFlag + } + + private[jdbc] val dbName = + if (StringUtils.isNotBlank(props.getProperty(DB_NAME))) props.getProperty(DB_NAME) + else "default" + + private val runningSQLStatements = new util.LinkedList[Statement] + + private var closed = false + + private var inited = false + + private[jdbc] val user = props.getProperty(USER) + + private[jdbc] val serverURL = props.getProperty("URL") + + private val labelMap: util.Map[String, AnyRef] = new util.HashMap[String, AnyRef] + + private val startupParams: util.Map[String, AnyRef] = new util.HashMap[String, AnyRef] + + private val runtimeParams: util.Map[String, AnyRef] = new util.HashMap[String, AnyRef] + + private[jdbc] def getEngineType: EngineTypeLabel = { + val engineType: EngineTypeLabel = + EngineTypeLabelCreator.createEngineTypeLabel(EngineType.TRINO.toString) + if (props.containsKey(PARAMS)) { + val params = props.getProperty(PARAMS) + if (params != null & params.length() > 0) { + params.split(PARAM_SPLIT).map(_.split(KV_SPLIT)).foreach { + case Array(k, v) if k.equals(UJESSQLDriver.ENGINE_TYPE) => + return EngineTypeLabelCreator.createEngineTypeLabel(v) + case _ => + } + } + } + engineType + } + + private[jdbc] def throwWhenClosed[T](op: => T): T = + if (isClosed) throw new LinkisSQLException(LinkisSQLErrorCode.CONNECTION_CLOSED) + else op + + private def createStatementAndAdd[T <: Statement](op: => T): T = throwWhenClosed { + + val statement = op + runningSQLStatements.add(statement) + if (!inited) { + inited = true + Utils.tryAndWarn(statement.execute(s"USE $dbName")) + } + statement + } + + def getProps: Properties = props + + def removeStatement(statement: LinkisSQLStatement): Unit = runningSQLStatements.remove(statement) + + override def createStatement(): Statement = createStatementAndAdd(new LinkisSQLStatement(this)) + + override def prepareStatement(sql: String): LinkisSQLPreparedStatement = { + val statement = createStatementAndAdd(new LinkisSQLPreparedStatement(this, sql)) + statement.clearQuery() + statement + } + + override def createStatement(resultSetType: Int, resultSetConcurrency: Int): Statement = { + if (resultSetConcurrency != ResultSet.CONCUR_READ_ONLY) { + throw new SQLException( + "Statement with resultset concurrency " + resultSetConcurrency + " is not supported", + "HYC00" + ) + } + if (resultSetType == ResultSet.TYPE_SCROLL_SENSITIVE) { + throw new SQLException( + "Statement with resultset type " + resultSetType + " is not supported", + "HYC00" + ) + } + createStatementAndAdd(new LinkisSQLStatement(this)) + } + + override def prepareStatement(sql: String, autoGeneratedKeys: Int): PreparedStatement = + prepareStatement(sql) + + override def prepareStatement( + sql: String, + resultSetType: Int, + resultSetConcurrency: Int + ): PreparedStatement = prepareStatement(sql) + + override def getMetaData: DatabaseMetaData = throwWhenClosed(new UJESSQLDatabaseMetaData(this)) + + override def close(): Unit = { + runningSQLStatements.asScala.foreach { statement => Utils.tryQuietly(statement.close()) } + closed = true + } + + override def isClosed: Boolean = closed + + override def setReadOnly(readOnly: Boolean): Unit = + throw new LinkisSQLException( + LinkisSQLErrorCode.NOSUPPORT_CONNECTION, + "setReadOnly not supported" + ) + + override def isReadOnly: Boolean = false + + override def setCatalog(catalog: String): Unit = throwWhenClosed() + + override def getCatalog: String = "" + + override def setTransactionIsolation(level: Int): Unit = {} + + override def getTransactionIsolation: Int = Connection.TRANSACTION_NONE + + override def getWarnings: SQLWarning = null + + override def clearWarnings(): Unit = {} + + override def setAutoCommit(autoCommit: Boolean): Unit = {} + + override def getAutoCommit: Boolean = true + + override def commit(): Unit = {} + + override def prepareCall(sql: String): CallableStatement = + throw new LinkisSQLException( + LinkisSQLErrorCode.NOSUPPORT_CONNECTION, + "prepareCall not supported" + ) + + override def rollback(): Unit = + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_CONNECTION, "rollback not supported") + + override def nativeSQL(sql: String): String = + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_CONNECTION, "nativeSQL not supported") + + override def prepareCall( + sql: String, + resultSetType: Int, + resultSetConcurrency: Int + ): CallableStatement = + throw new LinkisSQLException( + LinkisSQLErrorCode.NOSUPPORT_CONNECTION, + "prepareCall not supported" + ) + + override def getTypeMap: util.Map[String, Class[_]] = + throw new LinkisSQLException( + LinkisSQLErrorCode.NOSUPPORT_CONNECTION, + "getTypeMap not supported" + ) + + override def setTypeMap(map: util.Map[String, Class[_]]): Unit = + throw new LinkisSQLException( + LinkisSQLErrorCode.NOSUPPORT_CONNECTION, + "setTypeMap not supported" + ) + + override def setHoldability(holdability: Int): Unit = throw new LinkisSQLException( + LinkisSQLErrorCode.NOSUPPORT_CONNECTION, + "setHoldability not supported" + ) + + override def getHoldability: Int = 0 + + override def setSavepoint(): Savepoint = + throw new LinkisSQLException( + LinkisSQLErrorCode.NOSUPPORT_CONNECTION, + "setSavepoint not supported" + ) + + override def setSavepoint(name: String): Savepoint = + throw new LinkisSQLException( + LinkisSQLErrorCode.NOSUPPORT_CONNECTION, + "setSavepoint not supported" + ) + + override def rollback(savepoint: Savepoint): Unit = + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_CONNECTION, "rollback not supported") + + override def releaseSavepoint(savepoint: Savepoint): Unit = throw new LinkisSQLException( + LinkisSQLErrorCode.NOSUPPORT_CONNECTION, + "releaseSavepoint not supported" + ) + + override def createStatement( + resultSetType: Int, + resultSetConcurrency: Int, + resultSetHoldability: Int + ): Statement = + throw new LinkisSQLException( + LinkisSQLErrorCode.NOSUPPORT_CONNECTION, + "createStatement not supported" + ) + + override def prepareStatement( + sql: String, + resultSetType: Int, + resultSetConcurrency: Int, + resultSetHoldability: Int + ): PreparedStatement = + throw new LinkisSQLException( + LinkisSQLErrorCode.NOSUPPORT_CONNECTION, + "prepareStatement not supported" + ) + + override def prepareCall( + sql: String, + resultSetType: Int, + resultSetConcurrency: Int, + resultSetHoldability: Int + ): CallableStatement = + throw new LinkisSQLException( + LinkisSQLErrorCode.NOSUPPORT_CONNECTION, + "prepareCall not supported" + ) + + override def prepareStatement(sql: String, columnIndexes: Array[Int]): PreparedStatement = + throw new LinkisSQLException( + LinkisSQLErrorCode.NOSUPPORT_CONNECTION, + "prepareStatement not supported" + ) + + override def prepareStatement(sql: String, columnNames: Array[String]): PreparedStatement = + throw new LinkisSQLException( + LinkisSQLErrorCode.NOSUPPORT_CONNECTION, + "prepareStatement not supported" + ) + + override def createClob(): Clob = + throw new LinkisSQLException( + LinkisSQLErrorCode.NOSUPPORT_CONNECTION, + "createClob not supported" + ) + + override def createBlob(): Blob = + throw new LinkisSQLException( + LinkisSQLErrorCode.NOSUPPORT_CONNECTION, + "createBlob not supported" + ) + + override def createNClob(): NClob = + throw new LinkisSQLException( + LinkisSQLErrorCode.NOSUPPORT_CONNECTION, + "createNClob not supported" + ) + + override def createSQLXML(): SQLXML = + throw new LinkisSQLException( + LinkisSQLErrorCode.NOSUPPORT_CONNECTION, + "createSQLXML not supported" + ) + + override def isValid(timeout: Int): Boolean = true + + override def setClientInfo(name: String, value: String): Unit = + throw new LinkisSQLException( + LinkisSQLErrorCode.NOSUPPORT_CONNECTION, + "setClientInfo not supported" + ) + + override def setClientInfo(properties: Properties): Unit = + throw new LinkisSQLException( + LinkisSQLErrorCode.NOSUPPORT_CONNECTION, + "properties not supported" + ) + + override def getClientInfo(name: String): String = + throw new LinkisSQLException( + LinkisSQLErrorCode.NOSUPPORT_CONNECTION, + "getClientInfo not supported" + ) + + override def getClientInfo: Properties = + throw new LinkisSQLException( + LinkisSQLErrorCode.NOSUPPORT_CONNECTION, + "getClientInfo not supported" + ) + + override def createArrayOf(typeName: String, elements: Array[AnyRef]): sql.Array = + throw new LinkisSQLException( + LinkisSQLErrorCode.NOSUPPORT_CONNECTION, + "createArrayOf not supported" + ) + + override def createStruct(typeName: String, attributes: Array[AnyRef]): Struct = + throw new LinkisSQLException( + LinkisSQLErrorCode.NOSUPPORT_CONNECTION, + "createStruct not supported" + ) + + override def setSchema(schema: String): Unit = throwWhenClosed { + if (StringUtils.isBlank(schema)) { + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_STATEMENT, "schema is empty!") + } + createStatement().execute("use " + schema) + } + + override def getSchema: String = throwWhenClosed { + val resultSet = createStatement().executeQuery("SELECT current_database()") + if (!resultSet.next()) { + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_STATEMENT, "Get schema failed!") + } + resultSet.getString(1) + } + + override def abort(executor: Executor): Unit = + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_CONNECTION, "abort not supported") + + override def setNetworkTimeout(executor: Executor, milliseconds: Int): Unit = + throw new LinkisSQLException( + LinkisSQLErrorCode.NOSUPPORT_CONNECTION, + "setNetworkTimeout not supported" + ) + + override def getNetworkTimeout: Int = throw new LinkisSQLException( + LinkisSQLErrorCode.NOSUPPORT_CONNECTION, + "getNetworkTimeout not supported" + ) + + override def unwrap[T](iface: Class[T]): T = + throw new LinkisSQLException(LinkisSQLErrorCode.NOSUPPORT_CONNECTION, "unwrap not supported") + + override def isWrapperFor(iface: Class[_]): Boolean = + throw new LinkisSQLException( + LinkisSQLErrorCode.NOSUPPORT_CONNECTION, + "isWrapperFor not supported" + ) + + def addLabels(labels: util.Map[String, AnyRef]): Unit = { + labelMap.putAll(labels) + } + + def addStartUpParams(params: util.Map[String, AnyRef]): Unit = { + startupParams.putAll(params) + } + + def addRuntimeParams(params: util.Map[String, AnyRef]): Unit = { + runtimeParams.putAll(params) + } + + def engineToCodeType(engine: String): String = { + val runType = EngineType.mapStringToEngineType(engine) match { + case EngineType.SPARK => RunType.SQL + case EngineType.HIVE => RunType.HIVE + case EngineType.TRINO => RunType.TRINO_SQL + case EngineType.PRESTO => RunType.PRESTO_SQL + case EngineType.ELASTICSEARCH => RunType.ES_SQL + case EngineType.JDBC => RunType.JDBC + case EngineType.PYTHON => RunType.SHELL + case _ => RunType.SQL + } + runType.toString + } + + private[jdbc] def toSubmit(code: String): JobExecuteResult = { + val engineTypeLabel = getEngineType + labelMap.put(LabelKeyConstant.ENGINE_TYPE_KEY, engineTypeLabel.getStringValue) + labelMap.put(LabelKeyConstant.USER_CREATOR_TYPE_KEY, s"$user-$creator") + labelMap.put(LabelKeyConstant.CODE_TYPE_KEY, engineToCodeType(engineTypeLabel.getEngineType)) + + val jobSubmitAction = JobSubmitAction.builder + .addExecuteCode(code) + .setStartupParams(startupParams) + .setUser(user) + .addExecuteUser(user) + .setLabels(labelMap) + .setRuntimeParams(runtimeParams) + .setVariableMap(variableMap) + .build + + val result = ujesClient.submit(jobSubmitAction) + if (result.getStatus != 0) { + throw new SQLException(result.getMessage) + } + result + } + +} From 2052d08e4c763f02f5211ef108567f128ffdbfd6 Mon Sep 17 00:00:00 2001 From: peacewong Date: Tue, 6 Jun 2023 18:02:27 +0800 Subject: [PATCH 129/261] Add TicketID to Node --- .../am/restful/ECResourceInfoRestfulApi.java | 2 +- .../DefaultEngineConnPidCallbackService.java | 2 +- .../impl/ECResourceInfoServiceImpl.java | 2 +- .../service/impl/DefaultResourceManager.scala | 23 +++++--- .../linkis/manager/rm/utils/RMUtils.scala | 3 ++ .../manager/common/entity/node/AMEMNode.java | 12 +++++ .../common/entity/node/EngineNode.java | 4 -- .../common/entity/node/InfoRMNode.java | 12 +++++ .../manager/common/entity/node/Node.java | 4 ++ .../persistence/ECResourceInfoRecord.java | 10 ++++ .../entity/persistence/PersistenceNode.java | 10 ++++ .../persistence/PersistenceNodeEntity.java | 12 +++++ .../manager/common/utils/ManagerUtils.java | 3 +- .../linkis/manager/dao/NodeManagerMapper.java | 2 - .../impl/DefaultNodeManagerPersistence.java | 17 +++++- .../mapper/common/NodeManagerMapper.xml | 54 +++++++++++-------- linkis-dist/package/db/linkis_ddl.sql | 1 + .../upgrade/1.4.0_schema/mysql/linkis_ddl.sql | 18 +------ .../package/sbin/kill-process-by-port.sh | 22 ++++++++ 19 files changed, 157 insertions(+), 56 deletions(-) create mode 100644 linkis-dist/package/sbin/kill-process-by-port.sh diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/restful/ECResourceInfoRestfulApi.java b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/restful/ECResourceInfoRestfulApi.java index 60734a7056..df152b7008 100644 --- a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/restful/ECResourceInfoRestfulApi.java +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/restful/ECResourceInfoRestfulApi.java @@ -166,7 +166,7 @@ public Message queryEcrHistory( info -> { ECResourceInfoRecordVo ecrHistroryListVo = new ECResourceInfoRecordVo(); BeanUtils.copyProperties(info, ecrHistroryListVo); - ecrHistroryListVo.setEngineType(info.getLabelValue().split(",")[1].split("-")[0]); + ecrHistroryListVo.setEngineType(info.getEngineType()); ecrHistroryListVo.setUsedResource( ECResourceInfoUtils.getStringToMap(info.getUsedResource(), info)); ecrHistroryListVo.setReleasedResource( diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/service/impl/DefaultEngineConnPidCallbackService.java b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/service/impl/DefaultEngineConnPidCallbackService.java index e9a8290b05..3d199fe29c 100644 --- a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/service/impl/DefaultEngineConnPidCallbackService.java +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/service/impl/DefaultEngineConnPidCallbackService.java @@ -39,7 +39,7 @@ public class DefaultEngineConnPidCallbackService implements EngineConnPidCallbac @Receiver @Override public void dealPid(ResponseEngineConnPid protocol) { - // 设置pid + // set pid logger.info( "DefaultEngineConnPidCallbackService dealPid serviceInstance: [{}] pid: [{}]" + " ticketId: [{}]", diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/service/impl/ECResourceInfoServiceImpl.java b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/service/impl/ECResourceInfoServiceImpl.java index 953781586f..fc07e603ad 100644 --- a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/service/impl/ECResourceInfoServiceImpl.java +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/service/impl/ECResourceInfoServiceImpl.java @@ -177,7 +177,7 @@ public List> getECResourceInfoList( item.put("lastUnlockTimestamp", lastUnlockTimestamp); item.put("useResource", ECResourceInfoUtils.getStringToMap(usedResourceStr)); item.put("ecmInstance", latestRecord.getEcmInstance()); - String engineType = latestRecord.getLabelValue().split(",")[1].split("-")[0]; + String engineType = latestRecord.getEngineType(); item.put("engineType", engineType); if (StringUtils.isNotBlank(queueName)) { Map usedResourceMap = diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/rm/service/impl/DefaultResourceManager.scala b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/rm/service/impl/DefaultResourceManager.scala index b8f3f5b6fa..4e5fbec444 100644 --- a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/rm/service/impl/DefaultResourceManager.scala +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/rm/service/impl/DefaultResourceManager.scala @@ -52,6 +52,7 @@ import org.apache.linkis.manager.rm.{ ResourceInfo, ResultResource } +import org.apache.linkis.manager.rm.domain.RMLabelContainer import org.apache.linkis.manager.rm.entity.{LabelResourceMapping, ResourceOperationType} import org.apache.linkis.manager.rm.entity.ResourceOperationType.{LOCK, USED} import org.apache.linkis.manager.rm.exception.{RMErrorCode, RMLockFailedRetryException} @@ -312,22 +313,19 @@ class DefaultResourceManager extends ResourceManager with Logging with Initializ } { persistenceLocks.foreach(resourceLockService.unLock) } - // record engine locked resource - val tickedId = UUID.randomUUID().toString - resourceLogService.recordUserResourceAction( - labelContainer, - tickedId, - ChangeType.ENGINE_REQUEST, - resource.getLockedResource - ) + + // add ec node + val tickedId = RMUtils.getECTicketID val emNode = new AMEMNode emNode.setServiceInstance(labelContainer.getEMInstanceLabel.getServiceInstance) val engineNode = new AMEngineNode engineNode.setEMNode(emNode) engineNode.setServiceInstance(ServiceInstance(labelContainer.getEngineServiceName, tickedId)) engineNode.setNodeResource(resource) + engineNode.setTicketId(tickedId) nodeManagerPersistence.addEngineNode(engineNode) + // add labels val engineInstanceLabel = LabelBuilderFactoryContext.getLabelBuilderFactory.createLabel(classOf[EngineInstanceLabel]) engineInstanceLabel.setServiceName(labelContainer.getEngineServiceName) @@ -335,11 +333,20 @@ class DefaultResourceManager extends ResourceManager with Logging with Initializ nodeLabelService.addLabelToNode(engineNode.getServiceInstance, engineInstanceLabel) + // add ec resource labelResourceService.setEngineConnLabelResource( engineInstanceLabel, resource, labelContainer.getCombinedUserCreatorEngineTypeLabel.getStringValue ) + // record engine locked resource + labelContainer.getLabels.add(engineInstanceLabel) + resourceLogService.recordUserResourceAction( + labelContainer, + tickedId, + ChangeType.ENGINE_REQUEST, + resource.getLockedResource + ) val persistenceLabel = labelFactory.convertLabel(engineInstanceLabel, classOf[PersistenceLabel]) val persistenceEngineLabel = labelManagerPersistence.getLabelByKeyValue( diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/rm/utils/RMUtils.scala b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/rm/utils/RMUtils.scala index 2a35b7403d..a75b63b574 100644 --- a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/rm/utils/RMUtils.scala +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/rm/utils/RMUtils.scala @@ -29,6 +29,7 @@ import org.apache.linkis.manager.rm.restful.vo.UserResourceVo import org.apache.linkis.server.BDPJettyServerHelper import java.util +import java.util.UUID import scala.collection.JavaConverters.asScalaBufferConverter @@ -244,4 +245,6 @@ object RMUtils extends Logging { s" user ${resourceType}, requestResource : ${reqMsg} > availableResource : ${availMsg}, maxResource : ${maxMsg}." } + def getECTicketID: String = UUID.randomUUID().toString + } diff --git a/linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/java/org/apache/linkis/manager/common/entity/node/AMEMNode.java b/linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/java/org/apache/linkis/manager/common/entity/node/AMEMNode.java index c234bc00ad..34ba15601c 100644 --- a/linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/java/org/apache/linkis/manager/common/entity/node/AMEMNode.java +++ b/linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/java/org/apache/linkis/manager/common/entity/node/AMEMNode.java @@ -45,6 +45,8 @@ public class AMEMNode implements EMNode, ScoreServiceInstance { private String mark; private String identifier; + private String ticketId; + private NodeTaskInfo nodeTaskInfo; private NodeOverLoadInfo nodeOverLoadInfo; @@ -190,6 +192,16 @@ public void setNodeHealthyInfo(NodeHealthyInfo nodeHealthyInfo) { this.nodeHealthyInfo = nodeHealthyInfo; } + @Override + public String getTicketId() { + return ticketId; + } + + @Override + public void setTicketId(String ticketId) { + this.ticketId = ticketId; + } + @Override public String toString() { return "AMEMNode{" diff --git a/linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/java/org/apache/linkis/manager/common/entity/node/EngineNode.java b/linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/java/org/apache/linkis/manager/common/entity/node/EngineNode.java index 3a77112850..e3b8548bfc 100644 --- a/linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/java/org/apache/linkis/manager/common/entity/node/EngineNode.java +++ b/linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/java/org/apache/linkis/manager/common/entity/node/EngineNode.java @@ -26,8 +26,4 @@ public interface EngineNode extends AMNode, RMNode, LabelNode { String getLock(); void setLock(String lock); - - String getTicketId(); - - void setTicketId(String ticketId); } diff --git a/linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/java/org/apache/linkis/manager/common/entity/node/InfoRMNode.java b/linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/java/org/apache/linkis/manager/common/entity/node/InfoRMNode.java index 660ff0cf4c..c9b54bed42 100644 --- a/linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/java/org/apache/linkis/manager/common/entity/node/InfoRMNode.java +++ b/linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/java/org/apache/linkis/manager/common/entity/node/InfoRMNode.java @@ -40,6 +40,8 @@ public class InfoRMNode implements RMNode { private Date updateTime; + private String ticketId; + @Override public NodeResource getNodeResource() { return nodeResource; @@ -109,4 +111,14 @@ public Date getStartTime() { public void setStartTime(Date startTime) { this.startTime = startTime; } + + @Override + public String getTicketId() { + return ticketId; + } + + @Override + public void setTicketId(String ticketId) { + this.ticketId = ticketId; + } } diff --git a/linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/java/org/apache/linkis/manager/common/entity/node/Node.java b/linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/java/org/apache/linkis/manager/common/entity/node/Node.java index 1ff76a931f..a89c1552cc 100644 --- a/linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/java/org/apache/linkis/manager/common/entity/node/Node.java +++ b/linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/java/org/apache/linkis/manager/common/entity/node/Node.java @@ -48,4 +48,8 @@ public interface Node extends RequestProtocol { String getIdentifier(); void setIdentifier(String identifier); + + String getTicketId(); + + void setTicketId(String ticketId); } diff --git a/linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/java/org/apache/linkis/manager/common/entity/persistence/ECResourceInfoRecord.java b/linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/java/org/apache/linkis/manager/common/entity/persistence/ECResourceInfoRecord.java index 9d7c08158d..f4bd117172 100644 --- a/linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/java/org/apache/linkis/manager/common/entity/persistence/ECResourceInfoRecord.java +++ b/linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/java/org/apache/linkis/manager/common/entity/persistence/ECResourceInfoRecord.java @@ -20,6 +20,8 @@ import org.apache.linkis.manager.common.entity.resource.Resource; import org.apache.linkis.manager.common.utils.ResourceUtils; +import org.apache.commons.lang3.StringUtils; + import java.util.Date; public class ECResourceInfoRecord { @@ -87,6 +89,14 @@ public String getLabelValue() { return labelValue; } + public String getEngineType() { + if (StringUtils.isNotBlank(labelValue)) { + return labelValue.split(",")[1].split("-")[0]; + } else { + return ""; + } + } + public void setLabelValue(String labelValue) { this.labelValue = labelValue; } diff --git a/linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/java/org/apache/linkis/manager/common/entity/persistence/PersistenceNode.java b/linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/java/org/apache/linkis/manager/common/entity/persistence/PersistenceNode.java index 7302ffd63a..770a2e528a 100644 --- a/linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/java/org/apache/linkis/manager/common/entity/persistence/PersistenceNode.java +++ b/linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/java/org/apache/linkis/manager/common/entity/persistence/PersistenceNode.java @@ -29,6 +29,8 @@ public class PersistenceNode { /** identifier if mark equals "process", then identifier equals pid */ private String identifier; + private String ticketId; + private Date updateTime; private Date createTime; private String updator; @@ -50,6 +52,14 @@ public void setIdentifier(String identifier) { this.identifier = identifier; } + public String getTicketId() { + return ticketId; + } + + public void setTicketId(String ticketId) { + this.ticketId = ticketId; + } + public Integer getId() { return id; } diff --git a/linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/java/org/apache/linkis/manager/common/entity/persistence/PersistenceNodeEntity.java b/linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/java/org/apache/linkis/manager/common/entity/persistence/PersistenceNodeEntity.java index e119d5d82a..0a2452551a 100644 --- a/linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/java/org/apache/linkis/manager/common/entity/persistence/PersistenceNodeEntity.java +++ b/linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/java/org/apache/linkis/manager/common/entity/persistence/PersistenceNodeEntity.java @@ -35,6 +35,8 @@ public class PersistenceNodeEntity implements Node { private Date updateTime; + private String ticketId; + @Override public Date getUpdateTime() { return updateTime; @@ -99,6 +101,16 @@ public void setIdentifier(String identifier) { this.identifier = identifier; } + @Override + public String getTicketId() { + return ticketId; + } + + @Override + public void setTicketId(String ticketId) { + this.ticketId = ticketId; + } + public void setOwner(String owner) { this.owner = owner; } diff --git a/linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/java/org/apache/linkis/manager/common/utils/ManagerUtils.java b/linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/java/org/apache/linkis/manager/common/utils/ManagerUtils.java index 53137e4a8c..10f4b99efa 100644 --- a/linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/java/org/apache/linkis/manager/common/utils/ManagerUtils.java +++ b/linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/java/org/apache/linkis/manager/common/utils/ManagerUtils.java @@ -17,6 +17,7 @@ package org.apache.linkis.manager.common.utils; +import org.apache.linkis.common.utils.Utils; import org.apache.linkis.manager.common.conf.ManagerCommonConf; import org.apache.linkis.manager.label.builder.factory.LabelBuilderFactory; import org.apache.linkis.manager.label.builder.factory.LabelBuilderFactoryContext; @@ -44,7 +45,7 @@ public static String getAdminUser() { if (StringUtils.isNotBlank(ManagerCommonConf.DEFAULT_ADMIN().getValue())) { return ManagerCommonConf.DEFAULT_ADMIN().getValue(); } - return System.getProperty("user.name"); + return Utils.getJvmUser(); } public static Label persistenceLabelToRealLabel(Label label) { diff --git a/linkis-computation-governance/linkis-manager/linkis-manager-persistence/src/main/java/org/apache/linkis/manager/dao/NodeManagerMapper.java b/linkis-computation-governance/linkis-manager/linkis-manager-persistence/src/main/java/org/apache/linkis/manager/dao/NodeManagerMapper.java index 8c5f6b99ae..92d3032207 100644 --- a/linkis-computation-governance/linkis-manager/linkis-manager-persistence/src/main/java/org/apache/linkis/manager/dao/NodeManagerMapper.java +++ b/linkis-computation-governance/linkis-manager/linkis-manager-persistence/src/main/java/org/apache/linkis/manager/dao/NodeManagerMapper.java @@ -45,8 +45,6 @@ void updateNodeInstance( Integer getNodeInstanceId(@Param("instance") String instance); - Integer getIdByInstance(@Param("instance") String instance); - List getNodeInstanceIds(@Param("instances") List instances); PersistenceNode getNodeInstance(@Param("instance") String instance); diff --git a/linkis-computation-governance/linkis-manager/linkis-manager-persistence/src/main/java/org/apache/linkis/manager/persistence/impl/DefaultNodeManagerPersistence.java b/linkis-computation-governance/linkis-manager/linkis-manager-persistence/src/main/java/org/apache/linkis/manager/persistence/impl/DefaultNodeManagerPersistence.java index 4c6df1e16c..cb4ae406a9 100644 --- a/linkis-computation-governance/linkis-manager/linkis-manager-persistence/src/main/java/org/apache/linkis/manager/persistence/impl/DefaultNodeManagerPersistence.java +++ b/linkis-computation-governance/linkis-manager/linkis-manager-persistence/src/main/java/org/apache/linkis/manager/persistence/impl/DefaultNodeManagerPersistence.java @@ -78,6 +78,7 @@ public void addNodeInstance(Node node) throws PersistenceErrorException { persistenceNode.setName(node.getServiceInstance().getApplicationName()); persistenceNode.setOwner(node.getOwner()); persistenceNode.setMark(node.getMark()); + persistenceNode.setTicketId(node.getTicketId()); persistenceNode.setCreateTime(new Date()); persistenceNode.setUpdateTime(new Date()); persistenceNode.setCreator(node.getOwner()); @@ -156,6 +157,8 @@ public List getNodes(String owner) throws PersistenceErrorException { persistenceNodeEntity.setMark(persistenceNode.getMark()); persistenceNodeEntity.setOwner(persistenceNode.getOwner()); persistenceNodeEntity.setStartTime(persistenceNode.getCreateTime()); + persistenceNodeEntity.setIdentifier(persistenceNode.getIdentifier()); + persistenceNodeEntity.setTicketId(persistenceNode.getTicketId()); persistenceNodeEntitys.add(persistenceNodeEntity); } } @@ -174,6 +177,8 @@ public List getAllNodes() throws PersistenceErrorException { serviceInstance.setInstance(persistenceNode.getInstance()); persistenceNodeEntity.setServiceInstance(serviceInstance); persistenceNodeEntity.setMark(persistenceNode.getMark()); + persistenceNodeEntity.setIdentifier(persistenceNode.getIdentifier()); + persistenceNodeEntity.setTicketId(persistenceNode.getTicketId()); persistenceNodeEntity.setOwner(persistenceNode.getOwner()); persistenceNodeEntity.setStartTime(persistenceNode.getCreateTime()); persistenceNodeEntity.setUpdateTime(persistenceNode.getUpdateTime()); @@ -210,6 +215,8 @@ public Node getNode(ServiceInstance serviceInstance) { persistenceNodeEntity.setServiceInstance(serviceInstance); persistenceNodeEntity.setOwner(nodeInstances.getOwner()); persistenceNodeEntity.setMark(nodeInstances.getMark()); + persistenceNodeEntity.setIdentifier(nodeInstances.getIdentifier()); + persistenceNodeEntity.setTicketId(nodeInstances.getTicketId()); persistenceNodeEntity.setStartTime(nodeInstances.getCreateTime()); return persistenceNodeEntity; } @@ -218,7 +225,7 @@ public Node getNode(ServiceInstance serviceInstance) { public void addEngineNode(EngineNode engineNode) throws PersistenceErrorException { // insert engine(插入engine) addNodeInstance(engineNode); - // insert relationship,(插入关联关系，)todo 异常后续统一处理 + // insert relationship,(插入关联关系，) String engineNodeInstance = engineNode.getServiceInstance().getInstance(); if (null == engineNode.getEMNode()) { throw new PersistenceErrorException( @@ -256,6 +263,8 @@ public EngineNode getEngineNode(ServiceInstance serviceInstance) } amEngineNode.setOwner(engineNode.getOwner()); amEngineNode.setMark(engineNode.getMark()); + amEngineNode.setIdentifier(engineNode.getIdentifier()); + amEngineNode.setTicketId(engineNode.getTicketId()); amEngineNode.setStartTime(engineNode.getCreateTime()); PersistenceNode emNode = nodeManagerMapper.getEMNodeInstanceByEngineNode(serviceInstance.getInstance()); @@ -300,6 +309,8 @@ public List getEngineNodeByEM(ServiceInstance serviceInstance) amEngineNode.setServiceInstance(engineServiceInstance); amEngineNode.setOwner(engineNode.getOwner()); amEngineNode.setMark(engineNode.getMark()); + amEngineNode.setIdentifier(engineNode.getIdentifier()); + amEngineNode.setTicketId(engineNode.getTicketId()); amEngineNode.setStartTime(engineNode.getCreateTime()); amEngineNode.setEMNode(amEmNode); @@ -340,6 +351,8 @@ public List getEngineNodeByServiceInstance(List ser amEngineNode.setServiceInstance(serviceInstance); amEngineNode.setOwner(engineNode.getOwner()); amEngineNode.setMark(engineNode.getMark()); + amEngineNode.setIdentifier(engineNode.getIdentifier()); + amEngineNode.setTicketId(engineNode.getTicketId()); amEngineNode.setStartTime(engineNode.getCreateTime()); amEngineNodeList.add(amEngineNode); }); @@ -361,6 +374,8 @@ public List getNodesByOwnerList(List ownerlist) { serviceInstance.setInstance(persistenceNode.getInstance()); persistenceNodeEntity.setServiceInstance(serviceInstance); persistenceNodeEntity.setMark(persistenceNode.getMark()); + persistenceNodeEntity.setIdentifier(persistenceNode.getIdentifier()); + persistenceNodeEntity.setTicketId(persistenceNode.getTicketId()); persistenceNodeEntity.setOwner(persistenceNode.getOwner()); persistenceNodeEntity.setStartTime(persistenceNode.getCreateTime()); persistenceNodeEntitys.add(persistenceNodeEntity); diff --git a/linkis-computation-governance/linkis-manager/linkis-manager-persistence/src/main/resources/mapper/common/NodeManagerMapper.xml b/linkis-computation-governance/linkis-manager/linkis-manager-persistence/src/main/resources/mapper/common/NodeManagerMapper.xml index ecfaddc1fe..01409185ab 100644 --- a/linkis-computation-governance/linkis-manager/linkis-manager-persistence/src/main/resources/mapper/common/NodeManagerMapper.xml +++ b/linkis-computation-governance/linkis-manager/linkis-manager-persistence/src/main/resources/mapper/common/NodeManagerMapper.xml @@ -33,15 +33,39 @@ - INSERT INTO linkis_cg_manager_service_instance (instance, name, owner, mark, update_time + INSERT INTO linkis_cg_manager_service_instance (instance, name, owner, mark, ticketId, update_time , create_time, updator, creator) - VALUES (#{instance}, #{name}, #{owner}, #{mark}, #{updateTime} + VALUES (#{instance}, #{name}, #{owner}, #{mark}, #{ticketId}, #{updateTime} , #{createTime}, #{updator}, #{creator}) UPDATE linkis_cg_manager_service_instance - SET instance = #{persistenceNode.instance}, owner = #{persistenceNode.owner}, mark = #{persistenceNode.mark}, name = #{persistenceNode.name}, update_time = #{persistenceNode.updateTime}, updator = #{persistenceNode.updator}, creator = #{persistenceNode.creator} + SET + + instance = #{persistenceNode.instance}, + + + owner = #{persistenceNode.owner}, + + + mark = #{persistenceNode.mark}, + + + name = #{persistenceNode.name}, + + + update_time = #{persistenceNode.updateTime}, + + + updator = #{persistenceNode.updator}, + + + creator = #{persistenceNode.creator}, + + + identifier = #{persistenceNode.identifier} + WHERE instance = #{instance} @@ -61,12 +85,6 @@ FROM linkis_cg_manager_service_instance - - UPDATE linkis_cg_manager_service_instance - SET owner = #{persistenceNode.owner}, mark = #{persistenceNode.mark}, name = #{persistenceNode.name}, update_time = #{persistenceNode.updateTime}, create_time = #{persistenceNode.createTime}, updator = #{persistenceNode.updator}, creator = #{persistenceNode.creator} - WHERE instance = #{persistenceNode.instance} - - UPDATE linkis_cg_manager_service_instance SET @@ -97,12 +115,6 @@ WHERE instance = #{instance} - - diff --git a/linkis-dist/package/db/linkis_ddl.sql b/linkis-dist/package/db/linkis_ddl.sql index 371ace453a..ea0ca3d9a4 100644 --- a/linkis-dist/package/db/linkis_ddl.sql +++ b/linkis-dist/package/db/linkis_ddl.sql @@ -681,6 +681,7 @@ CREATE TABLE `linkis_cg_manager_service_instance` ( `owner` varchar(32) COLLATE utf8_bin DEFAULT NULL, `mark` varchar(32) COLLATE utf8_bin DEFAULT NULL, `identifier` varchar(32) COLLATE utf8_bin DEFAULT NULL, + `ticketId` varchar(255) COLLATE utf8_bin DEFAULT NULL, `update_time` datetime DEFAULT CURRENT_TIMESTAMP, `create_time` datetime DEFAULT CURRENT_TIMESTAMP, `updator` varchar(32) COLLATE utf8_bin DEFAULT NULL, diff --git a/linkis-dist/package/db/upgrade/1.4.0_schema/mysql/linkis_ddl.sql b/linkis-dist/package/db/upgrade/1.4.0_schema/mysql/linkis_ddl.sql index c3da20e590..0f1cda8793 100644 --- a/linkis-dist/package/db/upgrade/1.4.0_schema/mysql/linkis_ddl.sql +++ b/linkis-dist/package/db/upgrade/1.4.0_schema/mysql/linkis_ddl.sql @@ -15,19 +15,5 @@ * limitations under the License. */ -ALTER TABLE `linkis_ps_udf_manager` ADD COLUMN `create_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'create time'; -ALTER TABLE `linkis_ps_udf_manager` ADD COLUMN `update_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'update_time'; - -ALTER TABLE `linkis_ps_udf_shared_group` ADD COLUMN `create_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'create time'; -ALTER TABLE `linkis_ps_udf_shared_group` ADD COLUMN `update_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'update_time'; - -ALTER TABLE `linkis_ps_udf_shared_info` ADD COLUMN `create_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'create time'; -ALTER TABLE `linkis_ps_udf_shared_info` ADD COLUMN `update_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'update_time'; - -ALTER TABLE `linkis_ps_udf_user_load` ADD COLUMN `create_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'create time'; -ALTER TABLE `linkis_ps_udf_user_load` ADD COLUMN `update_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'update_time'; - - -ALTER TABLE `linkis_ps_udf_version` ADD COLUMN `update_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'update_time'; - -ALTER TABLE `linkis_ps_udf_tree` ADD CONSTRAINT `uniq_parent_name_uname_category` UNIQUE (`parent`,`name`, `user_name`, `category`); \ No newline at end of file +ALTER TABLE `linkis_cg_manager_service_instance` ADD COLUMN `identifier` varchar(32) COLLATE utf8_bin DEFAULT NULL; +ALTER TABLE `linkis_cg_manager_service_instance` ADD COLUMN `ticketId` varchar(255) COLLATE utf8_bin DEFAULT NULL; \ No newline at end of file diff --git a/linkis-dist/package/sbin/kill-process-by-port.sh b/linkis-dist/package/sbin/kill-process-by-port.sh new file mode 100644 index 0000000000..46dbf2ae81 --- /dev/null +++ b/linkis-dist/package/sbin/kill-process-by-port.sh @@ -0,0 +1,22 @@ +#!/usr/bin/env bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +port=$1 +pid=`ps -ef | grep server.port=$port | grep EngineConnServer | awk '{print $2}'` +echo "`date '+%Y-%m-%d %H:%M:%S'` Get port $port pid is $pid" +if [ "$pid" != "" ] +then + sh kill-process-by-pid.sh $pid +fi \ No newline at end of file From ad2fa2a4cdb1a94e765582661ba7415bdb582518 Mon Sep 17 00:00:00 2001 From: peacewong Date: Tue, 6 Jun 2023 18:05:28 +0800 Subject: [PATCH 130/261] Fixed ECM not cleaning up when starting EC exception --- .../common/utils/GovernanceConstant.scala | 1 - .../common/utils/GovernanceUtils.scala | 32 +++- .../common/utils/GovernanceConstantTest.scala | 2 - .../core/launch/ProcessEngineConnLaunch.scala | 9 +- .../ecm/server/listener/ECMReadyEvent.scala | 7 +- .../operator/EngineConnLogOperator.scala | 2 - .../AbstractEngineConnLaunchService.scala | 100 +++++------ .../service/impl/ECMListenerService.scala | 55 ++++++ .../impl/ProcessEngineConnLaunchService.scala | 165 +++++++----------- .../spring/ECMSpringConfiguration.scala | 29 +-- .../callback/service/EngineConnCallback.scala | 4 +- .../service/em/DefaultEMEngineService.scala | 15 +- 12 files changed, 228 insertions(+), 193 deletions(-) create mode 100644 linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/service/impl/ECMListenerService.scala diff --git a/linkis-computation-governance/linkis-computation-governance-common/src/main/scala/org/apache/linkis/governance/common/utils/GovernanceConstant.scala b/linkis-computation-governance/linkis-computation-governance-common/src/main/scala/org/apache/linkis/governance/common/utils/GovernanceConstant.scala index 52e7802164..54927a84df 100644 --- a/linkis-computation-governance/linkis-computation-governance-common/src/main/scala/org/apache/linkis/governance/common/utils/GovernanceConstant.scala +++ b/linkis-computation-governance/linkis-computation-governance-common/src/main/scala/org/apache/linkis/governance/common/utils/GovernanceConstant.scala @@ -31,5 +31,4 @@ object GovernanceConstant { val REQUEST_ENGINE_STATUS_BATCH_LIMIT = 500 - def RESULTSET_INDEX: String = "resultsetIndex" } diff --git a/linkis-computation-governance/linkis-computation-governance-common/src/main/scala/org/apache/linkis/governance/common/utils/GovernanceUtils.scala b/linkis-computation-governance/linkis-computation-governance-common/src/main/scala/org/apache/linkis/governance/common/utils/GovernanceUtils.scala index 301e295ef1..1710450ac5 100644 --- a/linkis-computation-governance/linkis-computation-governance-common/src/main/scala/org/apache/linkis/governance/common/utils/GovernanceUtils.scala +++ b/linkis-computation-governance/linkis-computation-governance-common/src/main/scala/org/apache/linkis/governance/common/utils/GovernanceUtils.scala @@ -73,13 +73,41 @@ object GovernanceUtils extends Logging { } } + def killProcessByPort(port: String, desc: String, isSudo: Boolean): Unit = { + val subProcessKillScriptPath = Configuration.getLinkisHome() + "/sbin/kill-process-by-port.sh" + if ( + StringUtils.isBlank(subProcessKillScriptPath) || !new File(subProcessKillScriptPath) + .exists() + ) { + logger.error(s"Failed to locate kill-script, $subProcessKillScriptPath not exist") + } else if (StringUtils.isNotBlank(port)) { + val cmd = if (isSudo) { + Array("sudo", "sh", subProcessKillScriptPath, String.valueOf(port)) + } else { + Array("sh", subProcessKillScriptPath, String.valueOf(port)) + } + logger.info( + s"Starting to kill process and sub-processes. desc: $desc Kill Command: " + cmd + .mkString(" ") + ) + + Utils.tryCatch { + val output = Utils.exec(cmd, 600 * 1000L) + logger.info(s"Kill Success! desc: $desc. msg:\n ${output}") + } { t => + logger.error(s"Kill error! desc: $desc.", t) + } + } + } + /** * find process id by port number * @param processPort * @return */ - def findProcessIdentifier(processPort: String) = { - val findCmd = "sudo lsof -t -i:" + processPort + def findProcessIdentifier(processPort: String): String = { + val findCmd = + "sudo netstat -tunlp | grep :" + processPort + " | awk '{print $7}' | awk -F/ '{print $1}'" val cmdList = new util.ArrayList[String] cmdList.add("bash") cmdList.add("-c") diff --git a/linkis-computation-governance/linkis-computation-governance-common/src/test/scala/org/apache/linkis/governance/common/utils/GovernanceConstantTest.scala b/linkis-computation-governance/linkis-computation-governance-common/src/test/scala/org/apache/linkis/governance/common/utils/GovernanceConstantTest.scala index 891d43c8b7..22f3cee233 100644 --- a/linkis-computation-governance/linkis-computation-governance-common/src/test/scala/org/apache/linkis/governance/common/utils/GovernanceConstantTest.scala +++ b/linkis-computation-governance/linkis-computation-governance-common/src/test/scala/org/apache/linkis/governance/common/utils/GovernanceConstantTest.scala @@ -31,7 +31,6 @@ class GovernanceConstantTest { val taskresourceversionstr = GovernanceConstant.TASK_RESOURCE_VERSION_STR val taskresourcefilenamestr = GovernanceConstant.TASK_RESOURCE_FILE_NAME_STR val requestenginestatusbatchlimit = GovernanceConstant.REQUEST_ENGINE_STATUS_BATCH_LIMIT - val resultsetindex = GovernanceConstant.RESULTSET_INDEX Assertions.assertEquals("source", tasksourcemapkey) Assertions.assertEquals("resources", taskresourcesstr) @@ -39,7 +38,6 @@ class GovernanceConstantTest { Assertions.assertEquals("version", taskresourceversionstr) Assertions.assertEquals("fileName", taskresourcefilenamestr) Assertions.assertTrue(500 == requestenginestatusbatchlimit.intValue()) - Assertions.assertEquals("resultsetIndex", resultsetindex) } diff --git a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-core/src/main/scala/org/apache/linkis/ecm/core/launch/ProcessEngineConnLaunch.scala b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-core/src/main/scala/org/apache/linkis/ecm/core/launch/ProcessEngineConnLaunch.scala index 91aa93e5fc..39b6b1e559 100644 --- a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-core/src/main/scala/org/apache/linkis/ecm/core/launch/ProcessEngineConnLaunch.scala +++ b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-core/src/main/scala/org/apache/linkis/ecm/core/launch/ProcessEngineConnLaunch.scala @@ -255,14 +255,15 @@ trait ProcessEngineConnLaunch extends EngineConnLaunch with Logging { } } + /** + * process exit code if process is null retur errorcode 10 + * @return + */ def processWaitFor: Int = { if (process != null) { process.waitFor } else { - throw new ECMCoreException( - CAN_NOT_GET_TERMINATED.getErrorCode, - CAN_NOT_GET_TERMINATED.getErrorDesc - ) + 10 } } diff --git a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/listener/ECMReadyEvent.scala b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/listener/ECMReadyEvent.scala index 97243d3cc0..db4ccea7f9 100644 --- a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/listener/ECMReadyEvent.scala +++ b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/listener/ECMReadyEvent.scala @@ -17,14 +17,13 @@ package org.apache.linkis.ecm.server.listener +import org.apache.linkis.ecm.core.engineconn.EngineConn import org.apache.linkis.ecm.core.listener.ECMEvent -import org.apache.linkis.governance.common.protocol.task.ResponseEngineConnPid -import org.apache.linkis.manager.common.entity.enumeration.NodeStatus -import org.apache.linkis.protocol.callback.{YarnAPPIdCallbackProtocol, YarnInfoCallbackProtocol} +import org.apache.linkis.manager.common.protocol.engine.EngineStopRequest case class ECMReadyEvent(params: Array[String]) extends ECMEvent case class ECMClosedEvent() extends ECMEvent -case class EngineConnLaunchStatusChangeEvent(tickedId: String, updateStatus: NodeStatus) +case class EngineConnStopEvent(engineConn: EngineConn, engineStopRequest: EngineStopRequest) extends ECMEvent diff --git a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/operator/EngineConnLogOperator.scala b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/operator/EngineConnLogOperator.scala index dd7f17356b..66327dadcf 100644 --- a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/operator/EngineConnLogOperator.scala +++ b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/operator/EngineConnLogOperator.scala @@ -39,8 +39,6 @@ import scala.util.matching.Regex class EngineConnLogOperator extends Operator with Logging { - private var localDirsHandleService: LocalDirsHandleService = _ - override def getNames: Array[String] = Array(EngineConnLogOperator.OPERATOR_NAME) override def apply(implicit parameters: Map[String, Any]): Map[String, Any] = { diff --git a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/service/impl/AbstractEngineConnLaunchService.scala b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/service/impl/AbstractEngineConnLaunchService.scala index 19581c99fe..9eb37f61b7 100644 --- a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/service/impl/AbstractEngineConnLaunchService.scala +++ b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/service/impl/AbstractEngineConnLaunchService.scala @@ -25,22 +25,25 @@ import org.apache.linkis.ecm.server.LinkisECMApplication import org.apache.linkis.ecm.server.conf.ECMConfiguration._ import org.apache.linkis.ecm.server.engineConn.DefaultEngineConn import org.apache.linkis.ecm.server.hook.ECMHook -import org.apache.linkis.ecm.server.listener.EngineConnLaunchStatusChangeEvent +import org.apache.linkis.ecm.server.listener.EngineConnStopEvent import org.apache.linkis.ecm.server.service.{EngineConnLaunchService, ResourceLocalizationService} import org.apache.linkis.ecm.server.util.ECMUtils import org.apache.linkis.governance.common.conf.GovernanceCommonConf -import org.apache.linkis.governance.common.utils.{JobUtils, LoggerUtils} +import org.apache.linkis.governance.common.utils.{ECPathUtils, JobUtils, LoggerUtils} +import org.apache.linkis.manager.common.constant.AMConstant import org.apache.linkis.manager.common.entity.enumeration.NodeStatus -import org.apache.linkis.manager.common.entity.enumeration.NodeStatus.Failed import org.apache.linkis.manager.common.entity.node.{AMEngineNode, EngineNode} -import org.apache.linkis.manager.common.protocol.engine.EngineConnStatusCallbackToAM +import org.apache.linkis.manager.common.protocol.engine.{ + EngineConnStatusCallbackToAM, + EngineStopRequest +} import org.apache.linkis.manager.engineplugin.common.launch.entity.EngineConnLaunchRequest +import org.apache.linkis.manager.label.utils.LabelUtil import org.apache.linkis.rpc.Sender import org.apache.commons.lang3.exception.ExceptionUtils -import scala.concurrent.{ExecutionContextExecutorService, Future} -import scala.util.{Failure, Success} +import scala.concurrent.ExecutionContextExecutorService abstract class AbstractEngineConnLaunchService extends EngineConnLaunchService with Logging { @@ -61,7 +64,7 @@ abstract class AbstractEngineConnLaunchService extends EngineConnLaunchService w } override def launchEngineConn(request: EngineConnLaunchRequest, duration: Long): EngineNode = { - // 1.创建engineConn和runner,launch 并设置基础属性 + // create engineConn/runner/launch val taskId = JobUtils.getJobIdFromStringMap(request.creationDesc.properties) LoggerUtils.setJobIdMDC(taskId) logger.info("TaskId: {} try to launch a new EngineConn with {}.", taskId: Any, request: Any) @@ -78,9 +81,9 @@ abstract class AbstractEngineConnLaunchService extends EngineConnLaunchService w conn.setStatus(NodeStatus.Starting) conn.setEngineConnInfo(new EngineConnInfo) conn.setEngineConnManagerEnv(launch.getEngineConnManagerEnv()) - // 2.资源本地化，并且设置ecm的env环境信息 + // get ec Resource getResourceLocalizationServie.handleInitEngineConnResources(request, conn) - // 4.run + // start ec Utils.tryCatch { beforeLaunch(request, conn, duration) runner.run() @@ -94,51 +97,12 @@ abstract class AbstractEngineConnLaunchService extends EngineConnLaunchService w case _ => } afterLaunch(request, conn, duration) - - val future = Future { - LoggerUtils.setJobIdMDC(taskId) - logger.info( - "TaskId: {} with request {} wait engineConn {} start", - Array(taskId, request, conn.getServiceInstance): _* - ) - Utils.tryFinally(waitEngineConnStart(request, conn, duration)) { - LoggerUtils.removeJobIdMDC() - } - } - - future onComplete { - case Failure(t) => - LoggerUtils.setJobIdMDC(taskId) - logger.error( - "TaskId: {} init {} failed. {} with request {}", - Array( - taskId, - conn.getServiceInstance, - conn.getEngineConnLaunchRunner.getEngineConnLaunch - .getEngineConnManagerEnv() - .engineConnWorkDir, - request - ): _* - ) - LinkisECMApplication.getContext.getECMSyncListenerBus.postToAll( - EngineConnLaunchStatusChangeEvent(conn.getTickedId, Failed) - ) - LoggerUtils.removeJobIdMDC() - case Success(_) => - LoggerUtils.setJobIdMDC(taskId) - logger.info( - "TaskId: {} init {} succeed. {} with request {}", - Array( - taskId, - conn.getServiceInstance, - conn.getEngineConnLaunchRunner.getEngineConnLaunch - .getEngineConnManagerEnv() - .engineConnWorkDir, - request - ): _* - ) - LoggerUtils.removeJobIdMDC() - } + logger.info( + "TaskId: {} with request {} wait engineConn {} start", + Array(taskId, request, conn.getServiceInstance): _* + ) + // start ec monitor thread + startEngineConnMonitorStart(request, conn, duration) } { t => logger.error( "TaskId: {} init {} failed, {}, with request {} now stop and delete it. message: {}", @@ -153,18 +117,29 @@ abstract class AbstractEngineConnLaunchService extends EngineConnLaunchService w t ): _* ) - conn.getEngineConnLaunchRunner.stop() Sender .getSender(MANAGER_SERVICE_NAME) .send( EngineConnStatusCallbackToAM( conn.getServiceInstance, - NodeStatus.ShuttingDown, + NodeStatus.Failed, " wait init failed , reason " + ExceptionUtils.getRootCauseMessage(t) ) ) - LinkisECMApplication.getContext.getECMSyncListenerBus.postToAll( - EngineConnLaunchStatusChangeEvent(conn.getTickedId, Failed) + conn.setStatus(NodeStatus.Failed) + val engineType = LabelUtil.getEngineType(request.labels) + val logPath = Utils.tryCatch(conn.getEngineConnManagerEnv.engineConnLogDirs) { t => + ECPathUtils.getECWOrkDirPathSuffix(request.user, request.ticketId, engineType) + } + val engineStopRequest = new EngineStopRequest + engineStopRequest.setEngineType(engineType) + engineStopRequest.setUser(request.user) + engineStopRequest.setIdentifier(conn.getPid) + engineStopRequest.setIdentifierType(AMConstant.PROCESS_MARK) + engineStopRequest.setLogDirSuffix(logPath) + engineStopRequest.setServiceInstance(conn.getServiceInstance) + LinkisECMApplication.getContext.getECMAsyncListenerBus.post( + EngineConnStopEvent(conn, engineStopRequest) ) LoggerUtils.removeJobIdMDC() throw t @@ -172,14 +147,17 @@ abstract class AbstractEngineConnLaunchService extends EngineConnLaunchService w LoggerUtils.removeJobIdMDC() val engineNode = new AMEngineNode() engineNode.setLabels(conn.getLabels) - engineNode.setServiceInstance(conn.getServiceInstance) engineNode.setOwner(request.user) - engineNode.setMark("process") + engineNode.setMark(AMConstant.PROCESS_MARK) engineNode } - def waitEngineConnStart(request: EngineConnLaunchRequest, conn: EngineConn, duration: Long): Unit + def startEngineConnMonitorStart( + request: EngineConnLaunchRequest, + conn: EngineConn, + duration: Long + ): Unit def createEngineConn: EngineConn = new DefaultEngineConn diff --git a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/service/impl/ECMListenerService.scala b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/service/impl/ECMListenerService.scala new file mode 100644 index 0000000000..209608d5cd --- /dev/null +++ b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/service/impl/ECMListenerService.scala @@ -0,0 +1,55 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.ecm.server.service.impl + +import org.apache.linkis.DataWorkCloudApplication +import org.apache.linkis.common.utils.Logging +import org.apache.linkis.ecm.core.listener.{ECMEvent, ECMEventListener} +import org.apache.linkis.ecm.server.listener.EngineConnStopEvent +import org.apache.linkis.ecm.server.service.EngineConnKillService +import org.apache.linkis.manager.common.entity.enumeration.NodeStatus + +class ECMListenerService extends ECMEventListener with Logging { + + private var engineConnKillService: EngineConnKillService = _ + + override def onEvent(event: ECMEvent): Unit = { + case EngineConnStopEvent(engineConn, engineStopRequest) => + if (NodeStatus.Failed == engineConn.getStatus) { + logger.info("deal stopEvent to kill ec {}", engineStopRequest) + engineConnKillService.dealEngineConnStop(engineStopRequest) + } else { + if (engineConnKillService.isInstanceOf[DefaultEngineConnKillService]) { + logger.info("deal stopEvent to kill yarn app {}", engineStopRequest) + engineConnKillService + .asInstanceOf[DefaultEngineConnKillService] + .killYarnAppIdOfOneEc(engineStopRequest) + } + } + case _ => + } + + def getEngineConnKillService(): EngineConnKillService = { + engineConnKillService + } + + def setEngineConnKillService(engineConnKillService: EngineConnKillService): Unit = { + this.engineConnKillService = engineConnKillService + } + +} diff --git a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/service/impl/ProcessEngineConnLaunchService.scala b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/service/impl/ProcessEngineConnLaunchService.scala index e6ab513f15..6c7626ae0e 100644 --- a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/service/impl/ProcessEngineConnLaunchService.scala +++ b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/service/impl/ProcessEngineConnLaunchService.scala @@ -19,31 +19,27 @@ package org.apache.linkis.ecm.server.service.impl import org.apache.linkis.common.conf.Configuration import org.apache.linkis.common.utils.Utils -import org.apache.linkis.ecm.core.conf.ECMErrorCode import org.apache.linkis.ecm.core.engineconn.EngineConn import org.apache.linkis.ecm.core.launch.ProcessEngineConnLaunch -import org.apache.linkis.ecm.errorcode.EngineconnServerErrorCodeSummary._ import org.apache.linkis.ecm.server.LinkisECMApplication import org.apache.linkis.ecm.server.conf.ECMConfiguration import org.apache.linkis.ecm.server.conf.ECMConfiguration.MANAGER_SERVICE_NAME -import org.apache.linkis.ecm.server.exception.ECMErrorException -import org.apache.linkis.ecm.server.listener.EngineConnLaunchStatusChangeEvent +import org.apache.linkis.ecm.server.listener.EngineConnStopEvent import org.apache.linkis.ecm.server.service.LocalDirsHandleService +import org.apache.linkis.governance.common.utils.{JobUtils, LoggerUtils} +import org.apache.linkis.manager.common.constant.AMConstant import org.apache.linkis.manager.common.entity.enumeration.NodeStatus -import org.apache.linkis.manager.common.entity.enumeration.NodeStatus._ -import org.apache.linkis.manager.common.protocol.engine.EngineConnStatusCallbackToAM +import org.apache.linkis.manager.common.protocol.engine.{ + EngineConnStatusCallbackToAM, + EngineStopRequest +} import org.apache.linkis.manager.engineplugin.common.launch.entity.EngineConnLaunchRequest import org.apache.linkis.manager.label.utils.LabelUtil import org.apache.linkis.rpc.Sender import org.apache.commons.io.IOUtils -import org.apache.commons.lang3.StringUtils -import org.apache.commons.lang3.exception.ExceptionUtils - -import java.util.concurrent.TimeUnit -import scala.concurrent.{Future, TimeoutException} -import scala.concurrent.duration.Duration +import scala.concurrent.Future abstract class ProcessEngineConnLaunchService extends AbstractEngineConnLaunchService { @@ -52,110 +48,83 @@ abstract class ProcessEngineConnLaunchService extends AbstractEngineConnLaunchSe def setLocalDirsHandleService(localDirsHandleService: LocalDirsHandleService): Unit = this.localDirsHandleService = localDirsHandleService - override def waitEngineConnStart( + override def startEngineConnMonitorStart( request: EngineConnLaunchRequest, conn: EngineConn, duration: Long ): Unit = { conn.getEngineConnLaunchRunner.getEngineConnLaunch match { case launch: ProcessEngineConnLaunch => - Utils.tryCatch { - // Set the pid of the shell script before the pid callBack returns - launch.getPid().foreach(conn.setPid) - processMonitorThread(conn, launch, duration) - } { case e: Throwable => - val logPath = Utils.tryCatch(conn.getEngineConnManagerEnv.engineConnLogDirs) { t => - localDirsHandleService.getEngineConnLogDir( - request.user, - request.ticketId, - LabelUtil.getEngineType(request.labels) - ) - } - val canRetry = e match { - case ecmError: ECMErrorException => - if (ECMErrorCode.EC_START_TIME_OUT == ecmError.getErrCode) { - true - } else if (StringUtils.isBlank(ecmError.getDesc)) { - logger.info("exception desc is null, can be retry") - true - } else { - false - } - case _ => false - } + launch.getPid().foreach(conn.setPid) + processMonitorThread(conn, launch) + case _ => + } + } + + private def processMonitorThread( + engineConn: EngineConn, + launch: ProcessEngineConnLaunch + ): Unit = { + Future { + val tickedId = engineConn.getTickedId + val errorMsg = new StringBuilder + val taskId = + JobUtils.getJobIdFromStringMap(launch.getEngineConnLaunchRequest.creationDesc.properties) + LoggerUtils.setJobIdMDC(taskId) + Utils.tryAndWarnMsg { + val iterator = + IOUtils.lineIterator(launch.getProcessInputStream, Configuration.BDP_ENCODING.getValue) + var count = 0 + val maxLen = ECMConfiguration.ENGINE_START_ERROR_MSG_MAX_LEN.getValue + while (launch.isAlive && iterator.hasNext && count < maxLen) { + val line = iterator.next() + errorMsg.append(line).append("\n") + count += 1 + } + val exitCode = launch.processWaitFor + val engineType = LabelUtil.getEngineType(launch.getEngineConnLaunchRequest.labels) + val logPath = Utils.tryCatch(engineConn.getEngineConnManagerEnv.engineConnLogDirs) { t => + localDirsHandleService.getEngineConnLogDir( + launch.getEngineConnLaunchRequest.user, + tickedId, + engineType + ) + } + if (exitCode != 0) { + val canRetry = if (errorMsg.isEmpty) true else false logger.warn( - s"Failed to init ${conn.getServiceInstance}, status shutting down, canRetry $canRetry, logPath $logPath", - e + s"Failed to start ec {}, status shutting down exit code {}, canRetry {}, logPath {}", + engineConn.getServiceInstance, + exitCode, + canRetry, + logPath ) Sender .getSender(MANAGER_SERVICE_NAME) .send( EngineConnStatusCallbackToAM( - conn.getServiceInstance, + engineConn.getServiceInstance, NodeStatus.ShuttingDown, - "Failed to start EngineConn, reason: " + ExceptionUtils.getRootCauseMessage( - e - ) + s"\n You can go to this path($logPath) to find the reason or ask the administrator for help", + "Failed to start EngineConn, reason: " + errorMsg + s"\n You can go to this path($logPath) to find the reason or ask the administrator for help", canRetry ) ) - throw e + engineConn.setStatus(NodeStatus.ShuttingDown) + } else { + engineConn.setStatus(NodeStatus.Success) } - case _ => - } - } - - private def processMonitorThread( - engineConn: EngineConn, - launch: ProcessEngineConnLaunch, - timeout: Long - ): Unit = { - val isCompleted: EngineConn => Boolean = engineConn => - engineConn.getStatus == Success || engineConn.getStatus == Failed - val tickedId = engineConn.getTickedId - val errorMsg = new StringBuilder - Future { - val iterator = - IOUtils.lineIterator(launch.getProcessInputStream, Configuration.BDP_ENCODING.getValue) - var count = 0 - val maxLen = ECMConfiguration.ENGINE_START_ERROR_MSG_MAX_LEN.getValue - while (!isCompleted(engineConn) && iterator.hasNext && count < maxLen) { - val line = iterator.next() - errorMsg.append(line).append("\n") - count += 1 - } - val exitCode = Option(launch.processWaitFor) - if (exitCode.exists(_ != 0)) { - logger.info(s"engine ${tickedId} process exit ") - LinkisECMApplication.getContext.getECMSyncListenerBus.postToAll( - EngineConnLaunchStatusChangeEvent(tickedId, ShuttingDown) - ) - } else { - LinkisECMApplication.getContext.getECMSyncListenerBus.postToAll( - EngineConnLaunchStatusChangeEvent(tickedId, Success) - ) - } - } - Utils.tryThrow( - Utils - .waitUntil(() => engineConn.getStatus != Starting, Duration(timeout, TimeUnit.MILLISECONDS)) - ) { - case e: TimeoutException => - throw new ECMErrorException( - EC_START_TIME_OUT.getErrorCode, - EC_START_TIME_OUT.getErrorDesc + s" $engineConn ." - ) - case e: InterruptedException => // 比如被ms cancel - throw new ECMErrorException( - EC_INTERRUPT_TIME_OUT.getErrorCode, - EC_INTERRUPT_TIME_OUT.getErrorDesc + s" $engineConn ." + val engineStopRequest = new EngineStopRequest + engineStopRequest.setEngineType(engineType) + engineStopRequest.setUser(launch.getEngineConnLaunchRequest.user) + engineStopRequest.setIdentifier(engineConn.getPid) + engineStopRequest.setIdentifierType(AMConstant.PROCESS_MARK) + engineStopRequest.setLogDirSuffix(logPath) + engineStopRequest.setServiceInstance(engineConn.getServiceInstance) + LinkisECMApplication.getContext.getECMAsyncListenerBus.post( + EngineConnStopEvent(engineConn, engineStopRequest) ) - case t: Throwable => - logger.error(s"unexpected error, now shutdown it.") - throw t - } - if (engineConn.getStatus == ShuttingDown) { - throw new ECMErrorException(EC_START_FAILED.getErrorCode, errorMsg.toString()) + } { s"EngineConns: ${engineConn.getServiceInstance} monitor Failed" } + LoggerUtils.removeJobIdMDC() } } diff --git a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/spring/ECMSpringConfiguration.scala b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/spring/ECMSpringConfiguration.scala index 9084d829f5..ec65cd885f 100644 --- a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/spring/ECMSpringConfiguration.scala +++ b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/spring/ECMSpringConfiguration.scala @@ -19,7 +19,7 @@ package org.apache.linkis.ecm.server.spring import org.apache.linkis.ecm.core.listener.ECMEventListener import org.apache.linkis.ecm.server.context.{DefaultECMContext, ECMContext} -import org.apache.linkis.ecm.server.service._ +import org.apache.linkis.ecm.server.service.{EngineConnKillService, _} import org.apache.linkis.ecm.server.service.impl._ import org.springframework.beans.factory.annotation.Autowired @@ -42,7 +42,6 @@ class ECMSpringConfiguration { @Bean @ConditionalOnMissingBean def getBmlResourceLocalizationService( - context: ECMContext, localDirsHandleService: LocalDirsHandleService ): ResourceLocalizationService = { val service: BmlResourceLocalizationService = new BmlResourceLocalizationService @@ -72,16 +71,16 @@ class ECMSpringConfiguration { @Bean @ConditionalOnMissingBean def getDefaultECMRegisterService(context: ECMContext): ECMRegisterService = { - implicit val service: DefaultECMRegisterService = new DefaultECMRegisterService - registerSyncListener(context) + val service: DefaultECMRegisterService = new DefaultECMRegisterService + registerSyncListener(context, service) service } @Bean @ConditionalOnMissingBean def getDefaultECMHealthService(context: ECMContext): ECMHealthService = { - implicit val service: DefaultECMHealthService = new DefaultECMHealthService - registerSyncListener(context) + val service: DefaultECMHealthService = new DefaultECMHealthService + registerSyncListener(context, service) service } @@ -93,15 +92,23 @@ class ECMSpringConfiguration { service } - private def registerSyncListener( + @Bean + @ConditionalOnMissingBean + def getECMListenerService( + engineConnKillService: EngineConnKillService, context: ECMContext - )(implicit listener: ECMEventListener): Unit = { + ): ECMListenerService = { + val service: ECMListenerService = new ECMListenerService + service.setEngineConnKillService(engineConnKillService) + registerASyncListener(context, service) + service + } + + private def registerSyncListener(context: ECMContext, listener: ECMEventListener): Unit = { context.getECMSyncListenerBus.addListener(listener) } - private def registerASyncListener( - context: ECMContext - )(implicit listener: ECMEventListener): Unit = { + private def registerASyncListener(context: ECMContext, listener: ECMEventListener): Unit = { context.getECMAsyncListenerBus.addListener(listener) } diff --git a/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/org/apache/linkis/engineconn/callback/service/EngineConnCallback.scala b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/org/apache/linkis/engineconn/callback/service/EngineConnCallback.scala index 1b629cf30c..da1f566467 100644 --- a/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/org/apache/linkis/engineconn/callback/service/EngineConnCallback.scala +++ b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/org/apache/linkis/engineconn/callback/service/EngineConnCallback.scala @@ -36,7 +36,7 @@ abstract class AbstractEngineConnStartUpCallback() extends EngineConnCallback wi protocol match { case protocol: EngineConnStatusCallback => if (protocol.status.equals(NodeStatus.Failed)) { - logger.error(s"protocol will send to lm: ${protocol}") + logger.error(s"EngineConn Start Failed protocol will send to lm: ${protocol}") } else { logger.info(s"protocol will send to lm: ${protocol}") } @@ -44,7 +44,7 @@ abstract class AbstractEngineConnStartUpCallback() extends EngineConnCallback wi } Sender .getSender(GovernanceCommonConf.ENGINE_APPLICATION_MANAGER_SPRING_NAME.getValue) - .ask(protocol) + .send(protocol) } } diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/service/em/DefaultEMEngineService.scala b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/service/em/DefaultEMEngineService.scala index 2ea0b8c5fb..f4af1c5550 100644 --- a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/service/em/DefaultEMEngineService.scala +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/service/em/DefaultEMEngineService.scala @@ -47,6 +47,7 @@ import org.apache.linkis.manager.rm.service.LabelResourceService import org.apache.linkis.manager.service.common.label.LabelFilter import org.apache.commons.collections.MapUtils +import org.apache.commons.lang3.StringUtils import org.springframework.beans.factory.annotation.Autowired import org.springframework.stereotype.Service @@ -116,14 +117,17 @@ class DefaultEMEngineService extends EMEngineService with Logging { engineStopRequest.setServiceInstance(engineNode.getServiceInstance) engineStopRequest.setIdentifierType(engineNode.getMark) engineStopRequest.setIdentifier(engineNode.getIdentifier) - val ecResourceInfo: ECResourceInfoRecord = - ecResourceInfoService.getECResourceInfoRecordByInstance( - engineNode.getServiceInstance.getInstance - ) + if (StringUtils.isNotBlank(engineNode.getTicketId)) { + ecResourceInfoService.getECResourceInfoRecord(engineNode.getTicketId) + } else { + ecResourceInfoService.getECResourceInfoRecordByInstance( + engineNode.getServiceInstance.getInstance + ) + } if (ecResourceInfo != null) { - engineStopRequest.setEngineType(ecResourceInfo.getLabelValue.split(",")(1).split("-")(0)) + engineStopRequest.setEngineType(ecResourceInfo.getEngineType()) engineStopRequest.setLogDirSuffix(ecResourceInfo.getLogDirSuffix) } else { if (engineNode.getLabels.isEmpty) { @@ -159,7 +163,6 @@ class DefaultEMEngineService extends EMEngineService with Logging { } emNodeManager.stopEngine(engineStopRequest, emNode) - // engineNodeManager.deleteEngineNode(engineNode) logger.info( s"EM ${emNode.getServiceInstance} finished to stop Engine ${engineNode.getServiceInstance}" ) From d618f588bbae8d7ba0fd2fbd2791ae9537beacfc Mon Sep 17 00:00:00 2001 From: peacewong Date: Tue, 6 Jun 2023 18:08:32 +0800 Subject: [PATCH 131/261] move File to java dir --- .../impl/DefaultEngineConnKillService.java | 285 ++++++++++++++++++ .../impl/DefaultEngineConnKillService.java | 213 ------------- 2 files changed, 285 insertions(+), 213 deletions(-) create mode 100644 linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/java/org/apache/linkis/ecm/server/service/impl/DefaultEngineConnKillService.java delete mode 100644 linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/service/impl/DefaultEngineConnKillService.java diff --git a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/java/org/apache/linkis/ecm/server/service/impl/DefaultEngineConnKillService.java b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/java/org/apache/linkis/ecm/server/service/impl/DefaultEngineConnKillService.java new file mode 100644 index 0000000000..aec7047ae5 --- /dev/null +++ b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/java/org/apache/linkis/ecm/server/service/impl/DefaultEngineConnKillService.java @@ -0,0 +1,285 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.ecm.server.service.impl; + +import org.apache.linkis.common.ServiceInstance; +import org.apache.linkis.common.utils.Utils; +import org.apache.linkis.ecm.server.conf.ECMConfiguration; +import org.apache.linkis.ecm.server.service.EngineConnKillService; +import org.apache.linkis.engineconn.common.conf.EngineConnConf; +import org.apache.linkis.governance.common.utils.GovernanceUtils; +import org.apache.linkis.manager.common.constant.AMConstant; +import org.apache.linkis.manager.common.protocol.engine.EngineStopRequest; +import org.apache.linkis.manager.common.protocol.engine.EngineStopResponse; +import org.apache.linkis.manager.common.protocol.engine.EngineSuicideRequest; +import org.apache.linkis.rpc.Sender; +import org.apache.linkis.rpc.message.annotation.Receiver; + +import org.apache.commons.io.IOUtils; +import org.apache.commons.lang3.StringUtils; + +import java.io.BufferedReader; +import java.io.File; +import java.io.FileNotFoundException; +import java.io.FileReader; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class DefaultEngineConnKillService implements EngineConnKillService { + + private static final Logger logger = LoggerFactory.getLogger(DefaultEngineConnKillService.class); + + private static final ThreadPoolExecutor ecYarnAppKillService = + Utils.newCachedThreadPool(10, "ECM-Kill-EC-Yarn-App", true); + + @Override + @Receiver + public EngineStopResponse dealEngineConnStop(EngineStopRequest engineStopRequest) { + logger.info("received EngineStopRequest " + engineStopRequest); + String pid = null; + if (AMConstant.PROCESS_MARK.equals(engineStopRequest.getIdentifierType()) + && StringUtils.isNotBlank(engineStopRequest.getIdentifier())) { + pid = engineStopRequest.getIdentifier(); + } + logger.info("dealEngineConnStop return pid: {}", pid); + EngineStopResponse response = new EngineStopResponse(); + if (StringUtils.isNotBlank(pid)) { + if (!killEngineConnByPid(pid, engineStopRequest.getServiceInstance())) { + response.setStopStatus(false); + response.setMsg( + "Kill engine " + engineStopRequest.getServiceInstance().toString() + " failed."); + } else { + response.setStopStatus(true); + response.setMsg( + "Kill engine " + engineStopRequest.getServiceInstance().toString() + " succeed."); + } + } else { + String processPort = engineStopRequest.getServiceInstance().getInstance().split(":")[1]; + logger.warn("Kill EC {} by port {}", engineStopRequest.getServiceInstance(), processPort); + if (!killEngineConnByPort(processPort, engineStopRequest.getServiceInstance())) { + response.setStopStatus(false); + response.setMsg( + "Kill engine " + engineStopRequest.getServiceInstance().toString() + " failed."); + } else { + response.setStopStatus(true); + response.setMsg( + "Kill engine " + engineStopRequest.getServiceInstance().toString() + " succeed."); + } + response.setStopStatus(false); + } + + // Requires default kill yarn appid + if (AMConstant.PROCESS_MARK.equals(engineStopRequest.getIdentifierType())) { + killYarnAppIdOfOneEc(engineStopRequest); + } + + if (!response.getStopStatus()) { + EngineSuicideRequest request = + new EngineSuicideRequest( + engineStopRequest.getServiceInstance(), engineStopRequest.getUser()); + try { + Sender.getSender(engineStopRequest.getServiceInstance()).send(request); + response.setStopStatus(true); + response.setMsg(response.getMsg() + " Now send suicide request to engine."); + } catch (Exception e) { + response.setMsg( + response.getMsg() + " Sended suicide request to engine error, " + e.getMessage()); + } + } + return response; + } + + public void killYarnAppIdOfOneEc(EngineStopRequest engineStopRequest) { + String logDirSuffix = engineStopRequest.getLogDirSuffix(); + ServiceInstance serviceInstance = engineStopRequest.getServiceInstance(); + String engineType = engineStopRequest.getEngineType(); + String engineConnInstance = serviceInstance.toString(); + String engineLogDir; + if (logDirSuffix.startsWith(ECMConfiguration.ENGINECONN_ROOT_DIR())) { + engineLogDir = logDirSuffix; + } else { + engineLogDir = ECMConfiguration.ENGINECONN_ROOT_DIR() + File.separator + logDirSuffix; + } + logger.info( + "try to kill yarn app ids in the engine of: [{}] engineLogDir: [{}]", + engineConnInstance, + engineLogDir); + final String errEngineLogPath = engineLogDir.concat(File.separator).concat("yarnApp"); + logger.info( + "try to parse the yarn app id from the engine err log file path: [{}]", errEngineLogPath); + File file = new File(errEngineLogPath); + if (file.exists()) { + ecYarnAppKillService.execute( + () -> { + BufferedReader in = null; + try { + in = new BufferedReader(new FileReader(errEngineLogPath)); + String line; + String regex = getYarnAppRegexByEngineType(engineType); + if (StringUtils.isBlank(regex)) { + return; + } + Pattern pattern = Pattern.compile(regex); + List appIds = new ArrayList<>(); + while ((line = in.readLine()) != null) { + if (StringUtils.isNotBlank(line)) { + Matcher mApp = pattern.matcher(line); + if (mApp.find()) { + String candidate1 = mApp.group(mApp.groupCount()); + if (!appIds.contains(candidate1)) { + appIds.add(candidate1); + } + } + } + } + GovernanceUtils.killYarnJobApp(appIds); + logger.info("finished kill yarn app ids in the engine of ({}).", engineConnInstance); + } catch (IOException ioEx) { + if (ioEx instanceof FileNotFoundException) { + logger.error("the engine log file {} not found.", errEngineLogPath); + } else { + logger.error( + "the engine log file parse failed. the reason is {}", ioEx.getMessage()); + } + } finally { + IOUtils.closeQuietly(in); + } + }); + } + } + + private String getYarnAppRegexByEngineType(String engineType) { + if (StringUtils.isBlank(engineType)) { + return ""; + } + String regex; + switch (engineType) { + case "spark": + case "shell": + regex = EngineConnConf.SPARK_ENGINE_CONN_YARN_APP_ID_PARSE_REGEX().getValue(); + break; + case "sqoop": + regex = EngineConnConf.SQOOP_ENGINE_CONN_YARN_APP_ID_PARSE_REGEX().getValue(); + break; + case "hive": + regex = EngineConnConf.HIVE_ENGINE_CONN_YARN_APP_ID_PARSE_REGEX().getValue(); + break; + default: + regex = ""; + } + return regex; + } + + private boolean killEngineConnByPid(String processId, ServiceInstance serviceInstance) { + logger.info("try to kill {} toString with pid({}).", serviceInstance.toString(), processId); + if (StringUtils.isNotBlank(processId)) { + if (ECMConfiguration.ECM_PROCESS_SCRIPT_KILL()) { + GovernanceUtils.killProcess(processId, serviceInstance.toString(), true); + } else { + killProcessByKillCmd(processId, serviceInstance.toString()); + } + return !isProcessAlive(processId); + } else { + logger.warn("cannot kill {} with empty pid.", serviceInstance); + return false; + } + } + + private boolean killEngineConnByPort(String port, ServiceInstance serviceInstance) { + logger.info("try to kill {} toString with port({}).", serviceInstance.toString(), port); + if (StringUtils.isNotBlank(port)) { + GovernanceUtils.killProcessByPort(port, serviceInstance.toString(), true); + return !isProcessAliveByPort(port); + } else { + logger.warn("cannot kill {} with empty port.", serviceInstance); + return false; + } + } + + private boolean isProcessAlive(String pid) { + String findCmd = + "ps -ef | grep " + + pid + + " | grep EngineConnServer | awk '{print \"exists_\"$2}' | grep " + + pid; + List cmdList = new ArrayList<>(); + cmdList.add("bash"); + cmdList.add("-c"); + cmdList.add(findCmd); + try { + String rs = Utils.exec(cmdList.toArray(new String[0]), 5000L); + return null != rs && rs.contains("exists_" + pid); + } catch (Exception e) { + logger.warn("Method isProcessAlive failed", e); + return false; + } + } + + private boolean isProcessAliveByPort(String port) { + String findCmd = + "ps -ef | grep server.port= " + + port + + " | grep EngineConnServer | awk -F \"server.port=\" '{print \"exists_\"$2}'"; + List cmdList = new ArrayList<>(); + cmdList.add("bash"); + cmdList.add("-c"); + cmdList.add(findCmd); + try { + String rs = Utils.exec(cmdList.toArray(new String[0]), 5000L); + return null != rs && rs.contains("exists_" + port); + } catch (Exception e) { + logger.warn("Method isProcessAlive failed", e); + return false; + } + } + + private void killProcessByKillCmd(String pid, String desc) { + String k15cmd = "sudo kill " + pid; + String k9cmd = "sudo kill -9 " + pid; + int tryNum = 0; + try { + while (isProcessAlive(pid) && tryNum <= 3) { + logger.info( + "{} still alive with pid({}), use shell command to kill it. try {}++", + desc, + pid, + tryNum++); + if (tryNum <= 3) { + Utils.exec(k15cmd.split(" "), 3000L); + } else { + logger.info( + "{} still alive with pid({}). try {}, use shell command to kill -9 it", + desc, + pid, + tryNum); + Utils.exec(k9cmd.split(" "), 3000L); + } + Thread.sleep(5000); + } + } catch (InterruptedException e) { + logger.error("Interrupted while killing engine {} with pid({})." + desc, pid); + } + } +} diff --git a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/service/impl/DefaultEngineConnKillService.java b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/service/impl/DefaultEngineConnKillService.java deleted file mode 100644 index 24f7797fba..0000000000 --- a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/service/impl/DefaultEngineConnKillService.java +++ /dev/null @@ -1,213 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.ecm.server.service.impl; - -import org.apache.commons.io.IOUtils; -import org.apache.linkis.common.ServiceInstance; -import org.apache.linkis.common.utils.Utils; -import org.apache.linkis.ecm.server.conf.ECMConfiguration; -import org.apache.linkis.ecm.server.service.EngineConnKillService; -import org.apache.linkis.engineconn.common.conf.EngineConnConf; -import org.apache.linkis.governance.common.utils.GovernanceUtils; -import org.apache.linkis.manager.common.protocol.engine.EngineStopRequest; -import org.apache.linkis.manager.common.protocol.engine.EngineStopResponse; -import org.apache.linkis.manager.common.protocol.engine.EngineSuicideRequest; -import org.apache.linkis.rpc.message.annotation.Receiver; -import org.apache.linkis.rpc.Sender; -import org.apache.commons.lang3.StringUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.BufferedReader; -import java.io.File; -import java.io.FileNotFoundException; -import java.io.FileReader; -import java.io.IOException; -import java.util.ArrayList; -import java.util.List; -import java.util.concurrent.ThreadPoolExecutor; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - -public class DefaultEngineConnKillService implements EngineConnKillService { - - private static final Logger logger = LoggerFactory.getLogger(DefaultEngineConnKillService.class); - - private static final ThreadPoolExecutor ecYarnAppKillService = Utils.newCachedThreadPool(10, "ECM-Kill-EC-Yarn-App", true); - - @Override - @Receiver - public EngineStopResponse dealEngineConnStop(EngineStopRequest engineStopRequest) { - logger.info("received EngineStopRequest " + engineStopRequest); - String pid = null; - if("process".equals(engineStopRequest.getIdentifierType()) && StringUtils.isNotBlank(engineStopRequest.getIdentifier())){ - pid = engineStopRequest.getIdentifier(); - }else { - String processPort = engineStopRequest.getServiceInstance().getInstance().split(":")[1]; - pid = GovernanceUtils.findProcessIdentifier(processPort); - } - - logger.info("dealEngineConnStop return pid: {}", pid); - EngineStopResponse response = new EngineStopResponse(); - if (StringUtils.isNotBlank(pid)) { - if(!killEngineConnByPid(pid, engineStopRequest.getServiceInstance())) { - response.setStopStatus(false); - response.setMsg("Kill engine " + engineStopRequest.getServiceInstance().toString() + " failed."); - } else { - response.setStopStatus(true); - response.setMsg("Kill engine " + engineStopRequest.getServiceInstance().toString() + " succeed."); - } - killYarnAppIdOfOneEc(engineStopRequest.getLogDirSuffix(), engineStopRequest.getServiceInstance(), - engineStopRequest.getEngineType()); - } else { - logger.warn("Cannot find engineConn pid, try kill with rpc"); - response.setStopStatus(false); - } - - if (!response.getStopStatus()) { - EngineSuicideRequest request = new EngineSuicideRequest(engineStopRequest.getServiceInstance(), engineStopRequest.getUser()); - try { - Sender.getSender(engineStopRequest.getServiceInstance()).send(request); - response.setStopStatus(true); - response.setMsg(response.getMsg() + " Now send suicide request to engine."); - } catch (Exception e) { - response.setMsg(response.getMsg() + " Sended suicide request to engine error, " + e.getMessage()); - } - } - return response; - } - - public void killYarnAppIdOfOneEc(String logDirSuffix, ServiceInstance serviceInstance, String engineType) { - String engineConnInstance = serviceInstance.toString(); - String engineLogDir = ECMConfiguration.ENGINECONN_ROOT_DIR() + File.separator + logDirSuffix; - logger.info("try to kill yarn app ids in the engine of: [{}] engineLogDir: [{}]", engineConnInstance, engineLogDir); - - final String errEngineLogPath = engineLogDir.concat(File.separator).concat("yarnApp"); - logger.info("try to parse the yarn app id from the engine err log file path: [{}]", errEngineLogPath); - File file = new File(errEngineLogPath); - if (file.exists()) { - ecYarnAppKillService.execute(() -> { - BufferedReader in = null; - try { - in = new BufferedReader(new FileReader(errEngineLogPath)); - String line; - String regex = getYarnAppRegexByEngineType(engineType); - if (StringUtils.isBlank(regex)) { - return; - } - Pattern pattern = Pattern.compile(regex); - List appIds = new ArrayList<>(); - while ((line = in.readLine()) != null) { - if (StringUtils.isNotBlank(line)) { - Matcher mApp = pattern.matcher(line); - if (mApp.find()) { - String candidate1 = mApp.group(mApp.groupCount()); - if (!appIds.contains(candidate1)) { - appIds.add(candidate1); - } - } - } - } - GovernanceUtils.killYarnJobApp(appIds); - logger.info("finished kill yarn app ids in the engine of ({}).", engineConnInstance); - } catch (IOException ioEx) { - if (ioEx instanceof FileNotFoundException) { - logger.error("the engine log file {} not found.", errEngineLogPath); - } else { - logger.error("the engine log file parse failed. the reason is {}", ioEx.getMessage()); - } - } finally { - IOUtils.closeQuietly(in); - } - }); - } - } - - private String getYarnAppRegexByEngineType(String engineType) { - if (StringUtils.isBlank(engineType)) { - return ""; - } - String regex; - switch (engineType) { - case "spark": - case "shell": - regex = EngineConnConf.SPARK_ENGINE_CONN_YARN_APP_ID_PARSE_REGEX().getValue(); - break; - case "sqoop": - regex = EngineConnConf.SQOOP_ENGINE_CONN_YARN_APP_ID_PARSE_REGEX().getValue(); - break; - case "hive": - regex = EngineConnConf.HIVE_ENGINE_CONN_YARN_APP_ID_PARSE_REGEX().getValue(); - break; - default: - regex = ""; - } - return regex; - } - - private boolean killEngineConnByPid(String processId, ServiceInstance serviceInstance) { - logger.info("try to kill {} toString with pid({}).", serviceInstance.toString(), processId); - if (StringUtils.isNotBlank(processId)) { - if (ECMConfiguration.ECM_PROCESS_SCRIPT_KILL()) { - GovernanceUtils.killProcess(processId, serviceInstance.toString(), true); - } else { - killProcessByKillCmd(processId, serviceInstance.toString()); - } - return !isProcessAlive(processId); - } else { - logger.warn("cannot kill {} with empty pid.", serviceInstance.toString()); - return false; - } - } - - private boolean isProcessAlive(String pid) { - String findCmd = "ps -ef | grep " + pid + " | grep EngineConnServer | awk '{print \"exists_\"$2}' | grep " + pid; - List cmdList = new ArrayList<>(); - cmdList.add("bash"); - cmdList.add("-c"); - cmdList.add(findCmd); - try { - String rs = Utils.exec(cmdList.toArray(new String[0]), 5000L); - return null != rs && rs.contains("exists_" + pid); - } catch (Exception e) { - // todo when thread catch exception , it should not be return false - logger.warn("Method isProcessAlive failed, " + e.getMessage()); - return false; - } - } - - private void killProcessByKillCmd(String pid, String desc ) { - String k15cmd = "sudo kill " + pid; - String k9cmd = "sudo kill -9 " + pid; - int tryNum = 0; - try { - while (isProcessAlive(pid) && tryNum <= 3) { - logger.info("{} still alive with pid({}), use shell command to kill it. try {}++", desc, pid, tryNum++); - if (tryNum <= 3) { - Utils.exec(k15cmd.split(" "), 3000L); - } else { - logger.info("{} still alive with pid({}). try {}, use shell command to kill -9 it", desc, pid, tryNum); - Utils.exec(k9cmd.split(" "), 3000L); - } - Thread.sleep(5000); - } - } catch (InterruptedException e) { - logger.error("Interrupted while killing engine {} with pid({})." + desc, pid); - } - } -} From 01d30c926cbb441bac5d07fec2e5d53d3b40688c Mon Sep 17 00:00:00 2001 From: peacewong Date: Tue, 6 Jun 2023 18:16:24 +0800 Subject: [PATCH 132/261] Because the information needs to be pushed to createEngine, there is no need to stop here, because create will stop --- ...efaultEngineConnStatusCallbackService.java | 22 +++++-------------- 1 file changed, 5 insertions(+), 17 deletions(-) diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/service/impl/DefaultEngineConnStatusCallbackService.java b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/service/impl/DefaultEngineConnStatusCallbackService.java index 12c45e2310..4a25487a5a 100644 --- a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/service/impl/DefaultEngineConnStatusCallbackService.java +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/service/impl/DefaultEngineConnStatusCallbackService.java @@ -25,10 +25,8 @@ import org.apache.linkis.manager.common.entity.metrics.AMNodeMetrics; import org.apache.linkis.manager.common.protocol.engine.EngineConnStatusCallback; import org.apache.linkis.manager.common.protocol.engine.EngineConnStatusCallbackToAM; -import org.apache.linkis.manager.common.protocol.engine.EngineStopRequest; import org.apache.linkis.manager.persistence.NodeMetricManagerPersistence; import org.apache.linkis.manager.service.common.metrics.MetricsConverter; -import org.apache.linkis.rpc.Sender$; import org.apache.linkis.rpc.message.annotation.Receiver; import org.apache.linkis.server.BDPJettyServerHelper; @@ -43,11 +41,14 @@ import java.util.Map; import com.fasterxml.jackson.core.JsonProcessingException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; @Service public class DefaultEngineConnStatusCallbackService implements EngineConnStatusCallbackService { - private org.slf4j.Logger logger = - org.slf4j.LoggerFactory.getLogger(DefaultEngineConnStatusCallbackService.class); + + private static final Logger logger = + LoggerFactory.getLogger(DefaultEngineConnStatusCallbackService.class); @Autowired private NodeMetricManagerPersistence nodeMetricManagerPersistence; @@ -65,19 +66,6 @@ public void dealEngineConnStatusCallback(EngineConnStatusCallback protocol) { protocol.serviceInstance(), protocol.status()); if (!NodeStatus.isAvailable(protocol.status())) { - EngineStopRequest engineStopRequest = new EngineStopRequest(); - engineStopRequest.setServiceInstance(protocol.serviceInstance()); - engineStopRequest.setUser("hadoop"); - try { - engineStopService.stopEngine( - engineStopRequest, Sender$.MODULE$.getSender(Sender$.MODULE$.getThisServiceInstance())); - } catch (Exception e) { - logger.warn( - "DefaultEngineConnStatusCallbackService stopEngine failed, serviceInstance:{}", - engineStopRequest.getServiceInstance(), - e); - } - dealEngineConnStatusCallbackToAM( new EngineConnStatusCallbackToAM( protocol.serviceInstance(), protocol.status(), protocol.initErrorMsg(), false)); From 3c4a0987f3a366b97afcb463ac315ac95b99c887 Mon Sep 17 00:00:00 2001 From: peacewong Date: Tue, 6 Jun 2023 20:33:39 +0800 Subject: [PATCH 133/261] Fix build error --- .../ecm/server/service/impl/ECMListenerService.scala | 3 +-- .../service/impl/ProcessEngineConnLaunchService.scala | 7 ++----- 2 files changed, 3 insertions(+), 7 deletions(-) diff --git a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/service/impl/ECMListenerService.scala b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/service/impl/ECMListenerService.scala index 209608d5cd..764a704887 100644 --- a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/service/impl/ECMListenerService.scala +++ b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/service/impl/ECMListenerService.scala @@ -17,7 +17,6 @@ package org.apache.linkis.ecm.server.service.impl -import org.apache.linkis.DataWorkCloudApplication import org.apache.linkis.common.utils.Logging import org.apache.linkis.ecm.core.listener.{ECMEvent, ECMEventListener} import org.apache.linkis.ecm.server.listener.EngineConnStopEvent @@ -28,7 +27,7 @@ class ECMListenerService extends ECMEventListener with Logging { private var engineConnKillService: EngineConnKillService = _ - override def onEvent(event: ECMEvent): Unit = { + override def onEvent(event: ECMEvent): Unit = event match { case EngineConnStopEvent(engineConn, engineStopRequest) => if (NodeStatus.Failed == engineConn.getStatus) { logger.info("deal stopEvent to kill ec {}", engineStopRequest) diff --git a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/service/impl/ProcessEngineConnLaunchService.scala b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/service/impl/ProcessEngineConnLaunchService.scala index 6c7626ae0e..94e866b4ce 100644 --- a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/service/impl/ProcessEngineConnLaunchService.scala +++ b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/service/impl/ProcessEngineConnLaunchService.scala @@ -93,11 +93,8 @@ abstract class ProcessEngineConnLaunchService extends AbstractEngineConnLaunchSe if (exitCode != 0) { val canRetry = if (errorMsg.isEmpty) true else false logger.warn( - s"Failed to start ec {}, status shutting down exit code {}, canRetry {}, logPath {}", - engineConn.getServiceInstance, - exitCode, - canRetry, - logPath + "Failed to start ec {}, status shutting down exit code {}, canRetry {}, logPath {}", + Array(engineConn.getServiceInstance, exitCode, canRetry, logPath): _* ) Sender .getSender(MANAGER_SERVICE_NAME) From 4aaf412813ba1d3d2b05b9478c7eb3f8c65b2bfe Mon Sep 17 00:00:00 2001 From: peacewong Date: Wed, 7 Jun 2023 12:11:18 +0800 Subject: [PATCH 134/261] Fix build error --- .../server/service/impl/ProcessEngineConnLaunchService.scala | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/service/impl/ProcessEngineConnLaunchService.scala b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/service/impl/ProcessEngineConnLaunchService.scala index 94e866b4ce..f5d2666a70 100644 --- a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/service/impl/ProcessEngineConnLaunchService.scala +++ b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/service/impl/ProcessEngineConnLaunchService.scala @@ -94,7 +94,10 @@ abstract class ProcessEngineConnLaunchService extends AbstractEngineConnLaunchSe val canRetry = if (errorMsg.isEmpty) true else false logger.warn( "Failed to start ec {}, status shutting down exit code {}, canRetry {}, logPath {}", - Array(engineConn.getServiceInstance, exitCode, canRetry, logPath): _* + engineConn.getServiceInstance: Any, + exitCode: Any, + canRetry: Any, + logPath: Any ) Sender .getSender(MANAGER_SERVICE_NAME) From 01f51a2bf29efa94cb3d4a49221387734595fd30 Mon Sep 17 00:00:00 2001 From: peacewong Date: Wed, 7 Jun 2023 14:22:01 +0800 Subject: [PATCH 135/261] Fix build error --- .../service/impl/ProcessEngineConnLaunchService.scala | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/service/impl/ProcessEngineConnLaunchService.scala b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/service/impl/ProcessEngineConnLaunchService.scala index f5d2666a70..f85b4754f5 100644 --- a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/service/impl/ProcessEngineConnLaunchService.scala +++ b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/service/impl/ProcessEngineConnLaunchService.scala @@ -93,11 +93,7 @@ abstract class ProcessEngineConnLaunchService extends AbstractEngineConnLaunchSe if (exitCode != 0) { val canRetry = if (errorMsg.isEmpty) true else false logger.warn( - "Failed to start ec {}, status shutting down exit code {}, canRetry {}, logPath {}", - engineConn.getServiceInstance: Any, - exitCode: Any, - canRetry: Any, - logPath: Any + s"Failed to start ec ${engineConn.getServiceInstance}, status shutting down exit code ${exitCode}, canRetry ${canRetry}, logPath ${logPath}" ) Sender .getSender(MANAGER_SERVICE_NAME) From f4f57487c597da0d4c9346604893feaf70adc025 Mon Sep 17 00:00:00 2001 From: peacewong Date: Fri, 9 Jun 2023 16:55:39 +0800 Subject: [PATCH 136/261] Fix Integration Test --- .../impl/DefaultEngineConnKillService.java | 2 +- .../impl/DefaultECMRegisterService.scala | 18 ++-- .../em/DefaultEMUnregisterService.scala | 5 -- .../service/impl/DefaultResourceManager.scala | 38 +++++--- .../DefaultNodeMetricManagerPersistence.java | 17 ++-- .../mapper/common/NodeManagerMapper.xml | 90 ++++++++++--------- .../package/sbin/kill-process-by-port.sh | 8 +- 7 files changed, 101 insertions(+), 77 deletions(-) diff --git a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/java/org/apache/linkis/ecm/server/service/impl/DefaultEngineConnKillService.java b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/java/org/apache/linkis/ecm/server/service/impl/DefaultEngineConnKillService.java index aec7047ae5..faf87acee8 100644 --- a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/java/org/apache/linkis/ecm/server/service/impl/DefaultEngineConnKillService.java +++ b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/java/org/apache/linkis/ecm/server/service/impl/DefaultEngineConnKillService.java @@ -223,7 +223,7 @@ private boolean isProcessAlive(String pid) { "ps -ef | grep " + pid + " | grep EngineConnServer | awk '{print \"exists_\"$2}' | grep " - + pid; + + pid + "|| true"; List cmdList = new ArrayList<>(); cmdList.add("bash"); cmdList.add("-c"); diff --git a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/service/impl/DefaultECMRegisterService.scala b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/service/impl/DefaultECMRegisterService.scala index eb9206c963..4c7807dad1 100644 --- a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/service/impl/DefaultECMRegisterService.scala +++ b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/service/impl/DefaultECMRegisterService.scala @@ -37,7 +37,9 @@ import java.util.Collections class DefaultECMRegisterService extends ECMRegisterService with ECMEventListener with Logging { - private implicit def readyEvent2RegisterECMRequest(event: ECMReadyEvent): RegisterEMRequest = { + private var unRegisterFlag = false + + private def readyEvent2RegisterECMRequest(event: ECMReadyEvent): RegisterEMRequest = { val request = new RegisterEMRequest val instance = Sender.getThisServiceInstance request.setUser(Utils.getJvmUser) @@ -50,14 +52,11 @@ class DefaultECMRegisterService extends ECMRegisterService with ECMEventListener private def getLabelsFromArgs(params: Array[String]): util.Map[String, AnyRef] = { import scala.collection.JavaConverters._ - val labelRegex = """label\.(.+)\.(.+)=(.+)""".r val labels = new util.HashMap[String, AnyRef]() - // TODO: magic labels.asScala += LabelKeyConstant.SERVER_ALIAS_KEY -> Collections.singletonMap( "alias", ENGINE_CONN_MANAGER_SPRING_NAME ) - // TODO: group by key labels } @@ -81,12 +80,12 @@ class DefaultECMRegisterService extends ECMRegisterService with ECMEventListener } override def onEvent(event: ECMEvent): Unit = event match { - case event: ECMReadyEvent => registerECM(event) - case event: ECMClosedEvent => unRegisterECM(event) + case event: ECMReadyEvent => registerECM(readyEvent2RegisterECMRequest(event)) + case event: ECMClosedEvent => unRegisterECM(closeEvent2StopECMRequest(event)) case _ => } - private implicit def closeEvent2StopECMRequest(event: ECMClosedEvent): StopEMRequest = { + private def closeEvent2StopECMRequest(event: ECMClosedEvent): StopEMRequest = { val request = new StopEMRequest val instance = Sender.getThisServiceInstance request.setUser(Utils.getJvmUser) @@ -114,7 +113,10 @@ class DefaultECMRegisterService extends ECMRegisterService with ECMEventListener override def unRegisterECM(request: StopEMRequest): Unit = { logger.info("start unRegister ecm") - Sender.getSender(MANAGER_SERVICE_NAME).send(request) + if (!unRegisterFlag) { + Sender.getSender(MANAGER_SERVICE_NAME).send(request) + } + unRegisterFlag = true } } diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/service/em/DefaultEMUnregisterService.scala b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/service/em/DefaultEMUnregisterService.scala index b54815092f..cdcc670fdd 100644 --- a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/service/em/DefaultEMUnregisterService.scala +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/service/em/DefaultEMUnregisterService.scala @@ -58,11 +58,6 @@ class DefaultEMUnregisterService extends EMUnregisterService with Logging { val emClearRequest = new EMInfoClearRequest emClearRequest.setEm(node) emClearRequest.setUser(stopEMRequest.getUser) - Utils.tryAndWarn(rmMessageService.dealWithStopEMRequest(stopEMRequest)) - // clear Label - val instanceLabelRemoveRequest = new NodeLabelRemoveRequest(node.getServiceInstance, false) - Utils.tryAndWarn(nodeLabelRemoveService.removeNodeLabel(instanceLabelRemoveRequest)) - // 此处需要先清理ECM再等待，避免ECM重启过快，导致ECM资源没清理干净 clearEMInstanceInfo(emClearRequest) logger.info(s" user ${stopEMRequest.getUser} finished to stop em ${stopEMRequest.getEm}") } diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/rm/service/impl/DefaultResourceManager.scala b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/rm/service/impl/DefaultResourceManager.scala index 4e5fbec444..c847da3c9a 100644 --- a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/rm/service/impl/DefaultResourceManager.scala +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/rm/service/impl/DefaultResourceManager.scala @@ -153,22 +153,40 @@ class DefaultResourceManager extends ResourceManager with Logging with Initializ eMInstanceLabel.setInstance(serviceInstance.getInstance) val emResource = labelResourceService.getLabelResource(eMInstanceLabel) + var registerResourceFlag = true if (emResource != null) { - logger.warn(s"${serviceInstance} has been registered, now update resource.") - if (!emResource.getResourceType.equals(resource.getResourceType)) { - throw new RMErrorException( - RMErrorCode.LABEL_DUPLICATED.getErrorCode, - MessageFormat.format( - RMErrorCode.LABEL_DUPLICATED.getErrorDesc, - serviceInstance, - emResource.getResourceType, - resource.getResourceType - ) + registerResourceFlag = false + logger.warn(s"${serviceInstance} has been registered, resource is ${emResource}.") + val leftResource = emResource.getLeftResource + if (leftResource != null && Resource.getZeroResource(leftResource) > leftResource) { + logger.warn( + s"${serviceInstance} has been registered, but left Resource <0 need to register resource." + ) + registerResourceFlag = true + } + val usedResource = emResource.getLockedResource + emResource.getUsedResource + if (usedResource > emResource.getMaxResource) { + logger.warn( + s"${serviceInstance} has been registered, but usedResource > MaxResource need to register resource." + ) + registerResourceFlag = true + } + + if (!(resource.getMaxResource == emResource.getMaxResource)) { + logger.warn( + s"${serviceInstance} has been registered, but Inconsistent newly registered resources need to register resource." ) + registerResourceFlag = true } } + + if (!registerResourceFlag) { + logger.warn(s"${serviceInstance} has been registered, skip register resource.") + return + } val lock = tryLockOneLabel(eMInstanceLabel, -1, Utils.getJvmUser) try { + labelResourceService.removeResourceByLabel(eMInstanceLabel) labelResourceService.setLabelResource( eMInstanceLabel, resource, diff --git a/linkis-computation-governance/linkis-manager/linkis-manager-persistence/src/main/java/org/apache/linkis/manager/persistence/impl/DefaultNodeMetricManagerPersistence.java b/linkis-computation-governance/linkis-manager/linkis-manager-persistence/src/main/java/org/apache/linkis/manager/persistence/impl/DefaultNodeMetricManagerPersistence.java index c6134cd410..03c0797918 100644 --- a/linkis-computation-governance/linkis-manager/linkis-manager-persistence/src/main/java/org/apache/linkis/manager/persistence/impl/DefaultNodeMetricManagerPersistence.java +++ b/linkis-computation-governance/linkis-manager/linkis-manager-persistence/src/main/java/org/apache/linkis/manager/persistence/impl/DefaultNodeMetricManagerPersistence.java @@ -18,6 +18,7 @@ package org.apache.linkis.manager.persistence.impl; import org.apache.linkis.common.ServiceInstance; +import org.apache.linkis.governance.common.conf.GovernanceCommonConf; import org.apache.linkis.manager.common.entity.enumeration.NodeStatus; import org.apache.linkis.manager.common.entity.metrics.NodeMetrics; import org.apache.linkis.manager.common.entity.node.Node; @@ -63,7 +64,6 @@ public void setNodeMetricManagerMapper(NodeMetricManagerMapper nodeMetricManager @Override public void addNodeMetrics(NodeMetrics nodeMetrics) throws PersistenceErrorException { - // 直接插入 NodeMetric即可 PersistenceNodeMetrics persistenceNodeMetrics = new PersistenceNodeMetrics(); persistenceNodeMetrics.setInstance(nodeMetrics.getServiceInstance().getInstance()); persistenceNodeMetrics.setHealthy(nodeMetrics.getHealthy()); @@ -72,7 +72,6 @@ public void addNodeMetrics(NodeMetrics nodeMetrics) throws PersistenceErrorExcep persistenceNodeMetrics.setStatus(nodeMetrics.getStatus()); persistenceNodeMetrics.setCreateTime(new Date()); persistenceNodeMetrics.setUpdateTime(new Date()); - // todo 异常信息后面统一处理 nodeMetricManagerMapper.addNodeMetrics(persistenceNodeMetrics); } @@ -84,7 +83,6 @@ public void addOrupdateNodeMetrics(NodeMetrics nodeMetrics) { return; } String instance = nodeMetrics.getServiceInstance().getInstance(); - // todo 异常信息后面统一处理 PersistenceNode node = nodeManagerMapper.getNodeInstance(instance); if (node == null) { logger.warn( @@ -94,7 +92,6 @@ public void addOrupdateNodeMetrics(NodeMetrics nodeMetrics) { return; } int isInstanceIdExist = nodeMetricManagerMapper.checkInstanceExist(instance); - // 是否存在 PersistenceNodeMetrics persistenceNodeMetrics = new PersistenceNodeMetrics(); if (isInstanceIdExist == 0) { persistenceNodeMetrics.setInstance(nodeMetrics.getServiceInstance().getInstance()); @@ -104,14 +101,20 @@ public void addOrupdateNodeMetrics(NodeMetrics nodeMetrics) { persistenceNodeMetrics.setStatus(nodeMetrics.getStatus()); persistenceNodeMetrics.setCreateTime(new Date()); persistenceNodeMetrics.setUpdateTime(new Date()); - // todo 异常信息后面统一处理 nodeMetricManagerMapper.addNodeMetrics(persistenceNodeMetrics); } else if (isInstanceIdExist == 1) { // ec node metircs report ignore update Shutingdown node (for case: asyn stop engine) PersistenceNodeMetrics oldMetrics = nodeMetricManagerMapper.getNodeMetricsByInstance(instance); - if (oldMetrics != null && NodeStatus.ShuttingDown.ordinal() <= oldMetrics.getStatus()) { + boolean isECM = + nodeMetrics + .getServiceInstance() + .getApplicationName() + .equalsIgnoreCase(GovernanceCommonConf.ENGINE_CONN_MANAGER_SPRING_NAME().getValue()); + if (!isECM + && oldMetrics != null + && NodeStatus.ShuttingDown.ordinal() <= oldMetrics.getStatus()) { logger.info( "ignore update status node:{} from:{} to status:{}", instance, @@ -129,7 +132,6 @@ public void addOrupdateNodeMetrics(NodeMetrics nodeMetrics) { persistenceNodeMetrics.setUpdateTime(new Date()); nodeMetricManagerMapper.updateNodeMetrics(persistenceNodeMetrics, instance); } else { - // 其他情况都不处理，打印个告警日志 } } @@ -144,7 +146,6 @@ public List getNodeMetrics(List nodes) instances.add(instance); } - // 根据 id 查 metric 信息 List persistenceNodeMetricsList = nodeMetricManagerMapper.getNodeMetricsByInstances(instances); diff --git a/linkis-computation-governance/linkis-manager/linkis-manager-persistence/src/main/resources/mapper/common/NodeManagerMapper.xml b/linkis-computation-governance/linkis-manager/linkis-manager-persistence/src/main/resources/mapper/common/NodeManagerMapper.xml index 01409185ab..3dca427c81 100644 --- a/linkis-computation-governance/linkis-manager/linkis-manager-persistence/src/main/resources/mapper/common/NodeManagerMapper.xml +++ b/linkis-computation-governance/linkis-manager/linkis-manager-persistence/src/main/resources/mapper/common/NodeManagerMapper.xml @@ -41,31 +41,32 @@ UPDATE linkis_cg_manager_service_instance - SET - - instance = #{persistenceNode.instance}, - - - owner = #{persistenceNode.owner}, - - - mark = #{persistenceNode.mark}, - - - name = #{persistenceNode.name}, - - - update_time = #{persistenceNode.updateTime}, - - - updator = #{persistenceNode.updator}, - - - creator = #{persistenceNode.creator}, - - - identifier = #{persistenceNode.identifier} - + + + instance = #{persistenceNode.instance}, + + + owner = #{persistenceNode.owner}, + + + mark = #{persistenceNode.mark}, + + + name = #{persistenceNode.name}, + + + update_time = #{persistenceNode.updateTime}, + + + updator = #{persistenceNode.updator}, + + + creator = #{persistenceNode.creator}, + + + identifier = #{persistenceNode.identifier}, + + WHERE instance = #{instance} @@ -87,25 +88,26 @@ UPDATE linkis_cg_manager_service_instance - SET - - mark = #{persistenceNode.mark}, - - - name = #{persistenceNode.name}, - - - update_time = #{persistenceNode.updateTime}, - - - updator = #{persistenceNode.updator}, - - - creator = #{persistenceNode.creator}, - - - identifier = #{persistenceNode.identifier} - + + + mark = #{persistenceNode.mark}, + + + name = #{persistenceNode.name}, + + + update_time = #{persistenceNode.updateTime}, + + + updator = #{persistenceNode.updator}, + + + creator = #{persistenceNode.creator}, + + + identifier = #{persistenceNode.identifier}, + + WHERE instance = #{persistenceNode.instance} diff --git a/linkis-dist/package/sbin/kill-process-by-port.sh b/linkis-dist/package/sbin/kill-process-by-port.sh index 46dbf2ae81..8be2f5904a 100644 --- a/linkis-dist/package/sbin/kill-process-by-port.sh +++ b/linkis-dist/package/sbin/kill-process-by-port.sh @@ -14,9 +14,15 @@ # limitations under the License. # port=$1 +shellDir=`dirname $0` +workDir=`cd ${shellDir}/..;pwd` +if [ "$LINKIS_HOME" = "" ] +then + LINKIS_HOME=$workDir +fi pid=`ps -ef | grep server.port=$port | grep EngineConnServer | awk '{print $2}'` echo "`date '+%Y-%m-%d %H:%M:%S'` Get port $port pid is $pid" if [ "$pid" != "" ] then - sh kill-process-by-pid.sh $pid + sh $LINKIS_HOME/sbin/kill-process-by-pid.sh $pid fi \ No newline at end of file From 9d6d0d0ed4a90fd5feb615a1e550d8e92f412578 Mon Sep 17 00:00:00 2001 From: peacewong Date: Sat, 10 Jun 2023 17:17:12 +0800 Subject: [PATCH 137/261] code optimize --- .../governance/common/utils/GovernanceUtils.scala | 9 +++++---- .../service/impl/DefaultEngineConnKillService.java | 6 +++--- .../impl/AbstractEngineConnLaunchService.scala | 8 ++------ .../impl/ProcessEngineConnLaunchService.scala | 3 +-- .../callback/service/EngineConnCallback.scala | 2 +- .../rm/service/impl/DefaultResourceManager.scala | 2 +- .../entity/persistence/ECResourceInfoRecord.java | 6 ++++++ .../impl/DefaultNodeMetricManagerPersistence.java | 12 ++++++------ ...process-by-port.sh => kill-ec-process-by-port.sh} | 0 9 files changed, 25 insertions(+), 23 deletions(-) rename linkis-dist/package/sbin/{kill-process-by-port.sh => kill-ec-process-by-port.sh} (100%) diff --git a/linkis-computation-governance/linkis-computation-governance-common/src/main/scala/org/apache/linkis/governance/common/utils/GovernanceUtils.scala b/linkis-computation-governance/linkis-computation-governance-common/src/main/scala/org/apache/linkis/governance/common/utils/GovernanceUtils.scala index 1710450ac5..ddcb17a3b2 100644 --- a/linkis-computation-governance/linkis-computation-governance-common/src/main/scala/org/apache/linkis/governance/common/utils/GovernanceUtils.scala +++ b/linkis-computation-governance/linkis-computation-governance-common/src/main/scala/org/apache/linkis/governance/common/utils/GovernanceUtils.scala @@ -73,8 +73,9 @@ object GovernanceUtils extends Logging { } } - def killProcessByPort(port: String, desc: String, isSudo: Boolean): Unit = { - val subProcessKillScriptPath = Configuration.getLinkisHome() + "/sbin/kill-process-by-port.sh" + def killECProcessByPort(port: String, desc: String, isSudo: Boolean): Unit = { + val subProcessKillScriptPath = + Configuration.getLinkisHome() + "/sbin/kill-ec-process-by-port.sh" if ( StringUtils.isBlank(subProcessKillScriptPath) || !new File(subProcessKillScriptPath) .exists() @@ -82,9 +83,9 @@ object GovernanceUtils extends Logging { logger.error(s"Failed to locate kill-script, $subProcessKillScriptPath not exist") } else if (StringUtils.isNotBlank(port)) { val cmd = if (isSudo) { - Array("sudo", "sh", subProcessKillScriptPath, String.valueOf(port)) + Array("sudo", "sh", subProcessKillScriptPath, port) } else { - Array("sh", subProcessKillScriptPath, String.valueOf(port)) + Array("sh", subProcessKillScriptPath, port) } logger.info( s"Starting to kill process and sub-processes. desc: $desc Kill Command: " + cmd diff --git a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/java/org/apache/linkis/ecm/server/service/impl/DefaultEngineConnKillService.java b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/java/org/apache/linkis/ecm/server/service/impl/DefaultEngineConnKillService.java index faf87acee8..ce8c760989 100644 --- a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/java/org/apache/linkis/ecm/server/service/impl/DefaultEngineConnKillService.java +++ b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/java/org/apache/linkis/ecm/server/service/impl/DefaultEngineConnKillService.java @@ -87,7 +87,6 @@ public EngineStopResponse dealEngineConnStop(EngineStopRequest engineStopRequest response.setMsg( "Kill engine " + engineStopRequest.getServiceInstance().toString() + " succeed."); } - response.setStopStatus(false); } // Requires default kill yarn appid @@ -210,7 +209,7 @@ private boolean killEngineConnByPid(String processId, ServiceInstance serviceIns private boolean killEngineConnByPort(String port, ServiceInstance serviceInstance) { logger.info("try to kill {} toString with port({}).", serviceInstance.toString(), port); if (StringUtils.isNotBlank(port)) { - GovernanceUtils.killProcessByPort(port, serviceInstance.toString(), true); + GovernanceUtils.killECProcessByPort(port, serviceInstance.toString(), true); return !isProcessAliveByPort(port); } else { logger.warn("cannot kill {} with empty port.", serviceInstance); @@ -223,7 +222,8 @@ private boolean isProcessAlive(String pid) { "ps -ef | grep " + pid + " | grep EngineConnServer | awk '{print \"exists_\"$2}' | grep " - + pid + "|| true"; + + pid + + "|| true"; List cmdList = new ArrayList<>(); cmdList.add("bash"); cmdList.add("-c"); diff --git a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/service/impl/AbstractEngineConnLaunchService.scala b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/service/impl/AbstractEngineConnLaunchService.scala index 9eb37f61b7..49e75fb6b8 100644 --- a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/service/impl/AbstractEngineConnLaunchService.scala +++ b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/service/impl/AbstractEngineConnLaunchService.scala @@ -102,7 +102,7 @@ abstract class AbstractEngineConnLaunchService extends EngineConnLaunchService w Array(taskId, request, conn.getServiceInstance): _* ) // start ec monitor thread - startEngineConnMonitorStart(request, conn, duration) + startEngineConnMonitorStart(request, conn) } { t => logger.error( "TaskId: {} init {} failed, {}, with request {} now stop and delete it. message: {}", @@ -153,11 +153,7 @@ abstract class AbstractEngineConnLaunchService extends EngineConnLaunchService w engineNode } - def startEngineConnMonitorStart( - request: EngineConnLaunchRequest, - conn: EngineConn, - duration: Long - ): Unit + def startEngineConnMonitorStart(request: EngineConnLaunchRequest, conn: EngineConn): Unit def createEngineConn: EngineConn = new DefaultEngineConn diff --git a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/service/impl/ProcessEngineConnLaunchService.scala b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/service/impl/ProcessEngineConnLaunchService.scala index f85b4754f5..dbb65b0059 100644 --- a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/service/impl/ProcessEngineConnLaunchService.scala +++ b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/service/impl/ProcessEngineConnLaunchService.scala @@ -50,8 +50,7 @@ abstract class ProcessEngineConnLaunchService extends AbstractEngineConnLaunchSe override def startEngineConnMonitorStart( request: EngineConnLaunchRequest, - conn: EngineConn, - duration: Long + conn: EngineConn ): Unit = { conn.getEngineConnLaunchRunner.getEngineConnLaunch match { case launch: ProcessEngineConnLaunch => diff --git a/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/org/apache/linkis/engineconn/callback/service/EngineConnCallback.scala b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/org/apache/linkis/engineconn/callback/service/EngineConnCallback.scala index da1f566467..d1eb83d391 100644 --- a/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/org/apache/linkis/engineconn/callback/service/EngineConnCallback.scala +++ b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/org/apache/linkis/engineconn/callback/service/EngineConnCallback.scala @@ -36,7 +36,7 @@ abstract class AbstractEngineConnStartUpCallback() extends EngineConnCallback wi protocol match { case protocol: EngineConnStatusCallback => if (protocol.status.equals(NodeStatus.Failed)) { - logger.error(s"EngineConn Start Failed protocol will send to lm: ${protocol}") + logger.error(s"EngineConn Start Failed protocol will send to LM: ${protocol}") } else { logger.info(s"protocol will send to lm: ${protocol}") } diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/rm/service/impl/DefaultResourceManager.scala b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/rm/service/impl/DefaultResourceManager.scala index c847da3c9a..0e6feb0ab4 100644 --- a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/rm/service/impl/DefaultResourceManager.scala +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/rm/service/impl/DefaultResourceManager.scala @@ -174,7 +174,7 @@ class DefaultResourceManager extends ResourceManager with Logging with Initializ if (!(resource.getMaxResource == emResource.getMaxResource)) { logger.warn( - s"${serviceInstance} has been registered, but Inconsistent newly registered resources need to register resource." + s"${serviceInstance} has been registered, but inconsistent newly registered resources need to register resource." ) registerResourceFlag = true } diff --git a/linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/java/org/apache/linkis/manager/common/entity/persistence/ECResourceInfoRecord.java b/linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/java/org/apache/linkis/manager/common/entity/persistence/ECResourceInfoRecord.java index f4bd117172..165606a090 100644 --- a/linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/java/org/apache/linkis/manager/common/entity/persistence/ECResourceInfoRecord.java +++ b/linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/java/org/apache/linkis/manager/common/entity/persistence/ECResourceInfoRecord.java @@ -89,6 +89,12 @@ public String getLabelValue() { return labelValue; } + /** + * label value is userCreator and engineTypeLabel,engineType is the second eg + * "hadoop-IDE,spark-2.4.3" + * + * @return + */ public String getEngineType() { if (StringUtils.isNotBlank(labelValue)) { return labelValue.split(",")[1].split("-")[0]; diff --git a/linkis-computation-governance/linkis-manager/linkis-manager-persistence/src/main/java/org/apache/linkis/manager/persistence/impl/DefaultNodeMetricManagerPersistence.java b/linkis-computation-governance/linkis-manager/linkis-manager-persistence/src/main/java/org/apache/linkis/manager/persistence/impl/DefaultNodeMetricManagerPersistence.java index 03c0797918..67f21a7fb5 100644 --- a/linkis-computation-governance/linkis-manager/linkis-manager-persistence/src/main/java/org/apache/linkis/manager/persistence/impl/DefaultNodeMetricManagerPersistence.java +++ b/linkis-computation-governance/linkis-manager/linkis-manager-persistence/src/main/java/org/apache/linkis/manager/persistence/impl/DefaultNodeMetricManagerPersistence.java @@ -108,13 +108,13 @@ public void addOrupdateNodeMetrics(NodeMetrics nodeMetrics) { nodeMetricManagerMapper.getNodeMetricsByInstance(instance); boolean isECM = - nodeMetrics - .getServiceInstance() - .getApplicationName() - .equalsIgnoreCase(GovernanceCommonConf.ENGINE_CONN_MANAGER_SPRING_NAME().getValue()); + nodeMetrics + .getServiceInstance() + .getApplicationName() + .equalsIgnoreCase(GovernanceCommonConf.ENGINE_CONN_MANAGER_SPRING_NAME().getValue()); if (!isECM - && oldMetrics != null - && NodeStatus.ShuttingDown.ordinal() <= oldMetrics.getStatus()) { + && oldMetrics != null + && NodeStatus.ShuttingDown.ordinal() <= oldMetrics.getStatus()) { logger.info( "ignore update status node:{} from:{} to status:{}", instance, diff --git a/linkis-dist/package/sbin/kill-process-by-port.sh b/linkis-dist/package/sbin/kill-ec-process-by-port.sh similarity index 100% rename from linkis-dist/package/sbin/kill-process-by-port.sh rename to linkis-dist/package/sbin/kill-ec-process-by-port.sh From 1627579406d9b6229639c47570f9c521e8f628da Mon Sep 17 00:00:00 2001 From: peacewong Date: Tue, 13 Jun 2023 15:59:20 +0800 Subject: [PATCH 138/261] Fix NPE bug #4621 --- .../apache/linkis/engineconn/common/conf/EngineConnConf.scala | 2 +- .../service/impl/DefaultEngineConnStatusCallbackService.java | 1 - .../linkis/manager/am/manager/DefaultEngineNodeManager.scala | 4 ++-- 3 files changed, 3 insertions(+), 4 deletions(-) diff --git a/linkis-computation-governance/linkis-engineconn/linkis-engineconn-common/src/main/scala/org/apache/linkis/engineconn/common/conf/EngineConnConf.scala b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-common/src/main/scala/org/apache/linkis/engineconn/common/conf/EngineConnConf.scala index 1518f00a6c..0d7f1cab05 100644 --- a/linkis-computation-governance/linkis-engineconn/linkis-engineconn-common/src/main/scala/org/apache/linkis/engineconn/common/conf/EngineConnConf.scala +++ b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-common/src/main/scala/org/apache/linkis/engineconn/common/conf/EngineConnConf.scala @@ -37,7 +37,7 @@ object EngineConnConf { val ENGINE_CONN_ONCE_HOOKS = CommonVars( "linkis.engine.connector.once.hooks", - "org.apache.linkis.engineconn.computation.executor.hook.ComputationEngineConnHook" + "org.apache.linkis.engineconn.once.executor.hook.OnceEngineConnHook" ) val ENGINE_LAUNCH_CMD_PARAMS_USER_KEY = diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/service/impl/DefaultEngineConnStatusCallbackService.java b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/service/impl/DefaultEngineConnStatusCallbackService.java index 4a25487a5a..7efeabd3e8 100644 --- a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/service/impl/DefaultEngineConnStatusCallbackService.java +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/service/impl/DefaultEngineConnStatusCallbackService.java @@ -70,7 +70,6 @@ public void dealEngineConnStatusCallback(EngineConnStatusCallback protocol) { new EngineConnStatusCallbackToAM( protocol.serviceInstance(), protocol.status(), protocol.initErrorMsg(), false)); } - logger.info("Finished to deal EngineConnStatusCallback: {}", protocol); } @Receiver diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/manager/DefaultEngineNodeManager.scala b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/manager/DefaultEngineNodeManager.scala index 098d6e5c74..2738e50bc1 100644 --- a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/manager/DefaultEngineNodeManager.scala +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/manager/DefaultEngineNodeManager.scala @@ -139,7 +139,7 @@ class DefaultEngineNodeManager extends EngineNodeManager with Logging { override def reuseEngine(engineNode: EngineNode): EngineNode = { val node = getEngineNodeInfo(engineNode) - if (!NodeStatus.isAvailable(node.getNodeStatus)) { + if (node == null || !NodeStatus.isAvailable(node.getNodeStatus)) { return null } if (!NodeStatus.isLocked(node.getNodeStatus)) { @@ -170,7 +170,7 @@ class DefaultEngineNodeManager extends EngineNodeManager with Logging { retryHandler.addRetryException(classOf[UndeclaredThrowableException]) val node = retryHandler.retry[EngineNode](getEngineNodeInfo(engineNode), "getEngineNodeInfo") // val node = getEngineNodeInfo(engineNode) - if (!NodeStatus.isAvailable(node.getNodeStatus)) { + if (node == null || !NodeStatus.isAvailable(node.getNodeStatus)) { return null } if (!NodeStatus.isLocked(node.getNodeStatus)) { From ba4e082854b7635c6789fbd48711ab965d4450c6 Mon Sep 17 00:00:00 2001 From: huangKai-2323 <62878639+huangKai-2323@users.noreply.github.com> Date: Wed, 7 Jun 2023 15:38:39 +0800 Subject: [PATCH 139/261] itsm json fix --- .../java/org/apache/linkis/configuration/entity/UserIpVo.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/linkis-public-enhancements/linkis-configuration/src/main/java/org/apache/linkis/configuration/entity/UserIpVo.java b/linkis-public-enhancements/linkis-configuration/src/main/java/org/apache/linkis/configuration/entity/UserIpVo.java index 05ec8046fd..77d2c67576 100644 --- a/linkis-public-enhancements/linkis-configuration/src/main/java/org/apache/linkis/configuration/entity/UserIpVo.java +++ b/linkis-public-enhancements/linkis-configuration/src/main/java/org/apache/linkis/configuration/entity/UserIpVo.java @@ -19,10 +19,12 @@ import java.util.Date; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import io.swagger.annotations.ApiModel; import io.swagger.annotations.ApiModelProperty; @ApiModel +@JsonIgnoreProperties(ignoreUnknown = true) public class UserIpVo { @ApiModelProperty("id") From 3b8322f8f7103280ae1d8ba54fa2acaacdfc6d8d Mon Sep 17 00:00:00 2001 From: ahaoyao <731753729@qq.com> Date: Thu, 1 Jun 2023 14:56:06 +0800 Subject: [PATCH 140/261] print real ip --- .../linkis/gateway/security/token/TokenAuthentication.scala | 5 ++++- .../gateway/springcloud/http/GatewayAuthorizationFilter.java | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/linkis-spring-cloud-services/linkis-service-gateway/linkis-gateway-core/src/main/scala/org/apache/linkis/gateway/security/token/TokenAuthentication.scala b/linkis-spring-cloud-services/linkis-service-gateway/linkis-gateway-core/src/main/scala/org/apache/linkis/gateway/security/token/TokenAuthentication.scala index 9a460eb52c..c5c801029a 100644 --- a/linkis-spring-cloud-services/linkis-service-gateway/linkis-gateway-core/src/main/scala/org/apache/linkis/gateway/security/token/TokenAuthentication.scala +++ b/linkis-spring-cloud-services/linkis-service-gateway/linkis-gateway-core/src/main/scala/org/apache/linkis/gateway/security/token/TokenAuthentication.scala @@ -93,7 +93,7 @@ object TokenAuthentication extends Logging { }) if (ok) { logger.info( - s"Token authentication succeed, uri: ${gatewayContext.getRequest.getRequestURI}, token: $token, tokenUser: $tokenUser." + s"Token authentication succeed, uri: ${gatewayContext.getRequest.getRequestURI}, token: $token, tokenUser: $tokenUser, host: $host." ) if (login) { logger.info( @@ -115,6 +115,9 @@ object TokenAuthentication extends Logging { } true } else { + logger.info( + s"Token authentication fail, uri: ${gatewayContext.getRequest.getRequestURI}, token: $token, tokenUser: $tokenUser, host: $host." + ) SecurityFilter.filterResponse(gatewayContext, authMsg) false } diff --git a/linkis-spring-cloud-services/linkis-service-gateway/linkis-spring-cloud-gateway/src/main/java/org/apache/linkis/gateway/springcloud/http/GatewayAuthorizationFilter.java b/linkis-spring-cloud-services/linkis-service-gateway/linkis-spring-cloud-gateway/src/main/java/org/apache/linkis/gateway/springcloud/http/GatewayAuthorizationFilter.java index 9bc1fa6017..71e53fb2b6 100644 --- a/linkis-spring-cloud-services/linkis-service-gateway/linkis-spring-cloud-gateway/src/main/java/org/apache/linkis/gateway/springcloud/http/GatewayAuthorizationFilter.java +++ b/linkis-spring-cloud-services/linkis-service-gateway/linkis-spring-cloud-gateway/src/main/java/org/apache/linkis/gateway/springcloud/http/GatewayAuthorizationFilter.java @@ -166,7 +166,7 @@ private Mono gatewayDeal( if (serviceInstance != null) { logger.info( "Client request ip: " - + gatewayContext.getRequest().getRemoteAddress() + + gatewayContext.getRequest().getRequestRealIpAddr() + " and uri: " + gatewayContext.getRequest().getRequestURI() + "GatewayRouter route requestUri: " From 8ab9742349d593d722e3246587ba4a3bd48a4ebd Mon Sep 17 00:00:00 2001 From: ahaoyao <731753729@qq.com> Date: Wed, 31 May 2023 11:28:53 +0800 Subject: [PATCH 141/261] Add exception logs when kill job interface --- .../apache/linkis/entrance/restful/EntranceRestfulApi.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/restful/EntranceRestfulApi.java b/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/restful/EntranceRestfulApi.java index eab67b2fe8..8d72100531 100644 --- a/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/restful/EntranceRestfulApi.java +++ b/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/restful/EntranceRestfulApi.java @@ -576,7 +576,8 @@ public Message killJobs( logger.error("kill job {} failed ", job.get().getId(), t); message = Message.error( - "An exception occurred while killing the job, kill failed(kill job的时候出现了异常，kill失败)"); + "An exception occurred while killing the job, kill failed(kill job的时候出现了异常，kill失败)", + t); message.setMethod("/api/entrance/" + id + "/kill"); message.setStatus(1); } @@ -660,7 +661,8 @@ public Message kill( logger.error("kill job {} failed ", job.get().getId(), t); message = Message.error( - "An exception occurred while killing the job, kill failed(kill job的时候出现了异常，kill失败)"); + "An exception occurred while killing the job, kill failed(kill job的时候出现了异常，kill失败)", + t); message.setMethod("/api/entrance/" + id + "/kill"); message.setStatus(1); } From 747bdcc6b0e3d540b8da16fa2a27fbbee306301d Mon Sep 17 00:00:00 2001 From: ahaoyao <731753729@qq.com> Date: Sun, 11 Jun 2023 22:56:12 +0800 Subject: [PATCH 142/261] delete setStatus --- .../org/apache/linkis/entrance/restful/EntranceRestfulApi.java | 1 - 1 file changed, 1 deletion(-) diff --git a/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/restful/EntranceRestfulApi.java b/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/restful/EntranceRestfulApi.java index 8d72100531..bf15fc8400 100644 --- a/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/restful/EntranceRestfulApi.java +++ b/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/restful/EntranceRestfulApi.java @@ -579,7 +579,6 @@ public Message killJobs( "An exception occurred while killing the job, kill failed(kill job的时候出现了异常，kill失败)", t); message.setMethod("/api/entrance/" + id + "/kill"); - message.setStatus(1); } } messages.add(message); From f9537ab6edaca462d300191138c9e2e5fa12db9b Mon Sep 17 00:00:00 2001 From: ahaoyao Date: Sun, 23 Apr 2023 14:14:38 +0800 Subject: [PATCH 143/261] properties values support the Chinese --- .../linkis/common/conf/BDPConfiguration.scala | 11 +++-- .../common/conf/BDPConfigurationTest.java | 41 +++++++++++++++++++ .../src/test/resources/linkis.properties | 16 ++++++++ 3 files changed, 65 insertions(+), 3 deletions(-) create mode 100644 linkis-commons/linkis-common/src/test/java/org/apache/linkis/common/conf/BDPConfigurationTest.java create mode 100644 linkis-commons/linkis-common/src/test/resources/linkis.properties diff --git a/linkis-commons/linkis-common/src/main/scala/org/apache/linkis/common/conf/BDPConfiguration.scala b/linkis-commons/linkis-common/src/main/scala/org/apache/linkis/common/conf/BDPConfiguration.scala index 55535e5336..14febab63a 100644 --- a/linkis-commons/linkis-common/src/main/scala/org/apache/linkis/common/conf/BDPConfiguration.scala +++ b/linkis-commons/linkis-common/src/main/scala/org/apache/linkis/common/conf/BDPConfiguration.scala @@ -22,7 +22,7 @@ import org.apache.linkis.common.utils.{Logging, Utils} import org.apache.commons.io.IOUtils import org.apache.commons.lang3.StringUtils -import java.io.{File, FileInputStream, InputStream, IOException} +import java.io._ import java.util.Properties import java.util.concurrent.TimeUnit import java.util.concurrent.locks.ReentrantReadWriteLock @@ -140,15 +140,20 @@ private[conf] object BDPConfiguration extends Logging { private def initConfig(config: Properties, filePath: String) { var inputStream: InputStream = null - + var reader: InputStreamReader = null + var buff: BufferedReader = null Utils.tryFinally { Utils.tryCatch { inputStream = new FileInputStream(filePath) - config.load(inputStream) + reader = new InputStreamReader(inputStream, "UTF-8") + buff = new BufferedReader(reader) + config.load(buff) } { case e: IOException => logger.error("Can't load " + filePath, e) } } { + IOUtils.closeQuietly(buff) + IOUtils.closeQuietly(reader) IOUtils.closeQuietly(inputStream) } } diff --git a/linkis-commons/linkis-common/src/test/java/org/apache/linkis/common/conf/BDPConfigurationTest.java b/linkis-commons/linkis-common/src/test/java/org/apache/linkis/common/conf/BDPConfigurationTest.java new file mode 100644 index 0000000000..5a025eb8b0 --- /dev/null +++ b/linkis-commons/linkis-common/src/test/java/org/apache/linkis/common/conf/BDPConfigurationTest.java @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.common.conf; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +/** BDPConfiguration Tester */ +public class BDPConfigurationTest { + + @Test + public void testGetOption() { + + Assertions.assertEquals( + "properties支持中文", + BDPConfiguration.getOption( + CommonVars.apply("linkis.jobhistory.error.msg.tip", "properties支持中文")) + .get()); + + Assertions.assertEquals( + "properties支持中文(默认)", + BDPConfiguration.getOption( + CommonVars.apply("linkis.jobhistory.error.msg.tip1", "properties支持中文(默认)")) + .get()); + } +} diff --git a/linkis-commons/linkis-common/src/test/resources/linkis.properties b/linkis-commons/linkis-common/src/test/resources/linkis.properties new file mode 100644 index 0000000000..230ad85a4e --- /dev/null +++ b/linkis-commons/linkis-common/src/test/resources/linkis.properties @@ -0,0 +1,16 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +linkis.jobhistory.error.msg.tip=properties支持中文 \ No newline at end of file From afc2999a7dc40c02d56e1952efe0895a62dc1201 Mon Sep 17 00:00:00 2001 From: huangKai-2323 <62878639+huangKai-2323@users.noreply.github.com> Date: Tue, 30 May 2023 16:17:31 +0800 Subject: [PATCH 144/261] update error code --- linkis-dist/package/db/linkis_dml.sql | 4 ++-- .../package/db/upgrade/1.4.0_schema/mysql/linkis_dml.sql | 9 +++++++++ 2 files changed, 11 insertions(+), 2 deletions(-) create mode 100644 linkis-dist/package/db/upgrade/1.4.0_schema/mysql/linkis_dml.sql diff --git a/linkis-dist/package/db/linkis_dml.sql b/linkis-dist/package/db/linkis_dml.sql index dffa9a7d76..b4dc1fa62c 100644 --- a/linkis-dist/package/db/linkis_dml.sql +++ b/linkis-dist/package/db/linkis_dml.sql @@ -380,8 +380,8 @@ INSERT INTO linkis_ps_error_code (error_code,error_desc,error_regex,error_type) INSERT INTO linkis_ps_error_code (error_code,error_desc,error_regex,error_type) VALUES ('01101','ECM资源不足，请联系管理员扩容','ECM resources are insufficient',0); INSERT INTO linkis_ps_error_code (error_code,error_desc,error_regex,error_type) VALUES ('01102','ECM 内存资源不足，请联系管理员扩容','ECM memory resources are insufficient',0); INSERT INTO linkis_ps_error_code (error_code,error_desc,error_regex,error_type) VALUES ('01103','ECM CPU资源不足，请联系管理员扩容','ECM CPU resources are insufficient',0); -INSERT INTO linkis_ps_error_code (error_code,error_desc,error_regex,error_type) VALUES ('01004','ECM 实例资源不足，请联系管理员扩容','ECM Insufficient number of instances',0); -INSERT INTO linkis_ps_error_code (error_code,error_desc,error_regex,error_type) VALUES ('01005','机器内存不足，请联系管理员扩容','Cannot allocate memory',0); +INSERT INTO linkis_ps_error_code (error_code,error_desc,error_regex,error_type) VALUES ('01104','ECM 实例资源不足，请联系管理员扩容','ECM Insufficient number of instances',0); +INSERT INTO linkis_ps_error_code (error_code,error_desc,error_regex,error_type) VALUES ('01105','机器内存不足，请联系管理员扩容','Cannot allocate memory',0); INSERT INTO linkis_ps_error_code (error_code,error_desc,error_regex,error_type) VALUES ('12001','队列CPU资源不足，可以调整Spark执行器个数','Queue CPU resources are insufficient',0); INSERT INTO linkis_ps_error_code (error_code,error_desc,error_regex,error_type) VALUES ('12002','队列内存资源不足，可以调整Spark执行器个数','Insufficient queue memory',0); diff --git a/linkis-dist/package/db/upgrade/1.4.0_schema/mysql/linkis_dml.sql b/linkis-dist/package/db/upgrade/1.4.0_schema/mysql/linkis_dml.sql new file mode 100644 index 0000000000..07e811210b --- /dev/null +++ b/linkis-dist/package/db/upgrade/1.4.0_schema/mysql/linkis_dml.sql @@ -0,0 +1,9 @@ +update linkis_ps_error_code set error_code = "01104" where error_regex ='ECM Insufficient number of instances'; +update linkis_ps_error_code set error_code = "01105" where error_regex ='Cannot allocate memory'; + + + + + + + From c53a0661139183d8c4bbc963548a678beaa3bab6 Mon Sep 17 00:00:00 2001 From: ahaoyao <731753729@qq.com> Date: Tue, 6 Jun 2023 20:22:01 +0800 Subject: [PATCH 145/261] Optimization of incorrect 500 scenario return results --- .../CustomErrorWebExceptionHandler.java | 99 +++++++++++++++++++ .../handler/ErrorHandlerConfiguration.java | 83 ++++++++++++++++ 2 files changed, 182 insertions(+) create mode 100644 linkis-spring-cloud-services/linkis-service-gateway/linkis-spring-cloud-gateway/src/main/java/org/apache/linkis/gateway/springcloud/handler/CustomErrorWebExceptionHandler.java create mode 100644 linkis-spring-cloud-services/linkis-service-gateway/linkis-spring-cloud-gateway/src/main/java/org/apache/linkis/gateway/springcloud/handler/ErrorHandlerConfiguration.java diff --git a/linkis-spring-cloud-services/linkis-service-gateway/linkis-spring-cloud-gateway/src/main/java/org/apache/linkis/gateway/springcloud/handler/CustomErrorWebExceptionHandler.java b/linkis-spring-cloud-services/linkis-service-gateway/linkis-spring-cloud-gateway/src/main/java/org/apache/linkis/gateway/springcloud/handler/CustomErrorWebExceptionHandler.java new file mode 100644 index 0000000000..f2bf8de354 --- /dev/null +++ b/linkis-spring-cloud-services/linkis-service-gateway/linkis-spring-cloud-gateway/src/main/java/org/apache/linkis/gateway/springcloud/handler/CustomErrorWebExceptionHandler.java @@ -0,0 +1,99 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.gateway.springcloud.handler; + +import org.springframework.boot.autoconfigure.web.ErrorProperties; +import org.springframework.boot.autoconfigure.web.ResourceProperties; +import org.springframework.boot.autoconfigure.web.reactive.error.DefaultErrorWebExceptionHandler; +import org.springframework.boot.web.error.ErrorAttributeOptions; +import org.springframework.boot.web.reactive.error.ErrorAttributes; +import org.springframework.context.ApplicationContext; +import org.springframework.core.annotation.MergedAnnotation; +import org.springframework.core.annotation.MergedAnnotations; +import org.springframework.http.HttpStatus; +import org.springframework.web.bind.annotation.ResponseStatus; +import org.springframework.web.reactive.function.server.*; +import org.springframework.web.server.ResponseStatusException; + +import java.util.LinkedHashMap; +import java.util.Map; + +import com.google.common.collect.Lists; + +public class CustomErrorWebExceptionHandler extends DefaultErrorWebExceptionHandler { + public CustomErrorWebExceptionHandler( + ErrorAttributes errorAttributes, + ResourceProperties resourceProperties, + ErrorProperties errorProperties, + ApplicationContext applicationContext) { + super(errorAttributes, resourceProperties, errorProperties, applicationContext); + } + + /** + * Get Exception Properties + * + * @param request + * @param options + */ + @Override + public Map getErrorAttributes( + ServerRequest request, ErrorAttributeOptions options) { + Map errorAttributes = new LinkedHashMap<>(); + errorAttributes.put("data", Lists.newArrayList()); + errorAttributes.put("method", request.path()); + Throwable error = getError(request); + MergedAnnotation responseStatusAnnotation = + MergedAnnotations.from(error.getClass(), MergedAnnotations.SearchStrategy.TYPE_HIERARCHY) + .get(ResponseStatus.class); + HttpStatus errorStatus = determineHttpStatus(error, responseStatusAnnotation); + errorAttributes.put("status", errorStatus.value()); + errorAttributes.put("message", errorStatus.getReasonPhrase()); + + return errorAttributes; + } + + private HttpStatus determineHttpStatus( + Throwable error, MergedAnnotation responseStatusAnnotation) { + if (error instanceof ResponseStatusException) { + return ((ResponseStatusException) error).getStatus(); + } + return responseStatusAnnotation + .getValue("code", HttpStatus.class) + .orElse(HttpStatus.INTERNAL_SERVER_ERROR); + } + + /** + * Specify the response processing method as JSON processing method + * + * @param errorAttributes + */ + @Override + public RouterFunction getRoutingFunction(ErrorAttributes errorAttributes) { + return RouterFunctions.route(RequestPredicates.all(), this::renderErrorResponse); + } + + /** + * Obtain the corresponding HttpStatus based on the status + * + * @param errorAttributes + */ + @Override + public int getHttpStatus(Map errorAttributes) { + return (int) errorAttributes.get("status"); + } +} diff --git a/linkis-spring-cloud-services/linkis-service-gateway/linkis-spring-cloud-gateway/src/main/java/org/apache/linkis/gateway/springcloud/handler/ErrorHandlerConfiguration.java b/linkis-spring-cloud-services/linkis-service-gateway/linkis-spring-cloud-gateway/src/main/java/org/apache/linkis/gateway/springcloud/handler/ErrorHandlerConfiguration.java new file mode 100644 index 0000000000..627b40e7d5 --- /dev/null +++ b/linkis-spring-cloud-services/linkis-service-gateway/linkis-spring-cloud-gateway/src/main/java/org/apache/linkis/gateway/springcloud/handler/ErrorHandlerConfiguration.java @@ -0,0 +1,83 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.gateway.springcloud.handler; + +import org.springframework.beans.factory.ObjectProvider; +import org.springframework.boot.autoconfigure.web.ResourceProperties; +import org.springframework.boot.autoconfigure.web.ServerProperties; +import org.springframework.boot.context.properties.EnableConfigurationProperties; +import org.springframework.boot.web.reactive.error.ErrorAttributes; +import org.springframework.boot.web.reactive.error.ErrorWebExceptionHandler; +import org.springframework.context.ApplicationContext; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.core.Ordered; +import org.springframework.core.annotation.Order; +import org.springframework.http.codec.ServerCodecConfigurer; +import org.springframework.web.reactive.result.view.ViewResolver; + +import java.util.Collections; +import java.util.List; + +/** + * 覆盖默认的异常处理 + * + * @author yinjihuan + */ +@Configuration +@EnableConfigurationProperties({ServerProperties.class, ResourceProperties.class}) +public class ErrorHandlerConfiguration { + + private final ServerProperties serverProperties; + + private final ApplicationContext applicationContext; + + private final ResourceProperties resourceProperties; + + private final List viewResolvers; + + private final ServerCodecConfigurer serverCodecConfigurer; + + public ErrorHandlerConfiguration( + ServerProperties serverProperties, + ResourceProperties resourceProperties, + ObjectProvider> viewResolversProvider, + ServerCodecConfigurer serverCodecConfigurer, + ApplicationContext applicationContext) { + this.serverProperties = serverProperties; + this.applicationContext = applicationContext; + this.resourceProperties = resourceProperties; + this.viewResolvers = viewResolversProvider.getIfAvailable(Collections::emptyList); + this.serverCodecConfigurer = serverCodecConfigurer; + } + + @Bean + @Order(Ordered.HIGHEST_PRECEDENCE) + public ErrorWebExceptionHandler errorWebExceptionHandler(ErrorAttributes errorAttributes) { + CustomErrorWebExceptionHandler exceptionHandler = + new CustomErrorWebExceptionHandler( + errorAttributes, + this.resourceProperties, + this.serverProperties.getError(), + this.applicationContext); + exceptionHandler.setViewResolvers(this.viewResolvers); + exceptionHandler.setMessageWriters(this.serverCodecConfigurer.getWriters()); + exceptionHandler.setMessageReaders(this.serverCodecConfigurer.getReaders()); + return exceptionHandler; + } +} From e38de081c75852e730ddff4490cb9ea91eedb461 Mon Sep 17 00:00:00 2001 From: ahaoyao <731753729@qq.com> Date: Wed, 7 Jun 2023 14:12:39 +0800 Subject: [PATCH 146/261] Modify comments --- .../springcloud/handler/ErrorHandlerConfiguration.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/linkis-spring-cloud-services/linkis-service-gateway/linkis-spring-cloud-gateway/src/main/java/org/apache/linkis/gateway/springcloud/handler/ErrorHandlerConfiguration.java b/linkis-spring-cloud-services/linkis-service-gateway/linkis-spring-cloud-gateway/src/main/java/org/apache/linkis/gateway/springcloud/handler/ErrorHandlerConfiguration.java index 627b40e7d5..3636193a94 100644 --- a/linkis-spring-cloud-services/linkis-service-gateway/linkis-spring-cloud-gateway/src/main/java/org/apache/linkis/gateway/springcloud/handler/ErrorHandlerConfiguration.java +++ b/linkis-spring-cloud-services/linkis-service-gateway/linkis-spring-cloud-gateway/src/main/java/org/apache/linkis/gateway/springcloud/handler/ErrorHandlerConfiguration.java @@ -35,9 +35,9 @@ import java.util.List; /** - * 覆盖默认的异常处理 + * Overwrite default exception handling * - * @author yinjihuan + * @author ahaoyao */ @Configuration @EnableConfigurationProperties({ServerProperties.class, ResourceProperties.class}) From 5779f11a4bd19c9317c4de49be908f4c626c3118 Mon Sep 17 00:00:00 2001 From: ahaoyao <731753729@qq.com> Date: Sun, 11 Jun 2023 22:41:54 +0800 Subject: [PATCH 147/261] remove author info --- .../gateway/springcloud/handler/ErrorHandlerConfiguration.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/linkis-spring-cloud-services/linkis-service-gateway/linkis-spring-cloud-gateway/src/main/java/org/apache/linkis/gateway/springcloud/handler/ErrorHandlerConfiguration.java b/linkis-spring-cloud-services/linkis-service-gateway/linkis-spring-cloud-gateway/src/main/java/org/apache/linkis/gateway/springcloud/handler/ErrorHandlerConfiguration.java index 3636193a94..d4d47530fa 100644 --- a/linkis-spring-cloud-services/linkis-service-gateway/linkis-spring-cloud-gateway/src/main/java/org/apache/linkis/gateway/springcloud/handler/ErrorHandlerConfiguration.java +++ b/linkis-spring-cloud-services/linkis-service-gateway/linkis-spring-cloud-gateway/src/main/java/org/apache/linkis/gateway/springcloud/handler/ErrorHandlerConfiguration.java @@ -36,8 +36,6 @@ /** * Overwrite default exception handling - * - * @author ahaoyao */ @Configuration @EnableConfigurationProperties({ServerProperties.class, ResourceProperties.class}) From ff9d1537513b6e4699b9caa05491599e6986467f Mon Sep 17 00:00:00 2001 From: ahaoyao <731753729@qq.com> Date: Wed, 14 Jun 2023 14:57:07 +0800 Subject: [PATCH 148/261] Implement custom DefaultErrorAttributes --- .../handler/CustomErrorAttributes.java | 65 ++++++++++++ .../CustomErrorWebExceptionHandler.java | 99 ------------------- .../handler/ErrorHandlerConfiguration.java | 81 --------------- 3 files changed, 65 insertions(+), 180 deletions(-) create mode 100644 linkis-spring-cloud-services/linkis-service-gateway/linkis-spring-cloud-gateway/src/main/java/org/apache/linkis/gateway/springcloud/handler/CustomErrorAttributes.java delete mode 100644 linkis-spring-cloud-services/linkis-service-gateway/linkis-spring-cloud-gateway/src/main/java/org/apache/linkis/gateway/springcloud/handler/CustomErrorWebExceptionHandler.java delete mode 100644 linkis-spring-cloud-services/linkis-service-gateway/linkis-spring-cloud-gateway/src/main/java/org/apache/linkis/gateway/springcloud/handler/ErrorHandlerConfiguration.java diff --git a/linkis-spring-cloud-services/linkis-service-gateway/linkis-spring-cloud-gateway/src/main/java/org/apache/linkis/gateway/springcloud/handler/CustomErrorAttributes.java b/linkis-spring-cloud-services/linkis-service-gateway/linkis-spring-cloud-gateway/src/main/java/org/apache/linkis/gateway/springcloud/handler/CustomErrorAttributes.java new file mode 100644 index 0000000000..4cbafa6625 --- /dev/null +++ b/linkis-spring-cloud-services/linkis-service-gateway/linkis-spring-cloud-gateway/src/main/java/org/apache/linkis/gateway/springcloud/handler/CustomErrorAttributes.java @@ -0,0 +1,65 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.gateway.springcloud.handler; + +import org.springframework.boot.web.error.ErrorAttributeOptions; +import org.springframework.boot.web.reactive.error.DefaultErrorAttributes; +import org.springframework.core.annotation.MergedAnnotation; +import org.springframework.core.annotation.MergedAnnotations; +import org.springframework.http.HttpStatus; +import org.springframework.stereotype.Component; +import org.springframework.web.bind.annotation.ResponseStatus; +import org.springframework.web.reactive.function.server.ServerRequest; +import org.springframework.web.server.ResponseStatusException; + +import java.util.HashMap; +import java.util.Map; + +import com.google.common.collect.Lists; + +@Component +public class CustomErrorAttributes extends DefaultErrorAttributes { + + @Override + public Map getErrorAttributes( + ServerRequest request, ErrorAttributeOptions options) { + Throwable throwable = this.getError(request); + MergedAnnotation responseStatusAnnotation = + MergedAnnotations.from( + throwable.getClass(), MergedAnnotations.SearchStrategy.TYPE_HIERARCHY) + .get(ResponseStatus.class); + HttpStatus errorStatus = determineHttpStatus(throwable, responseStatusAnnotation); + Map map = new HashMap<>(); + map.put("method", request.path()); + map.put("status", errorStatus.value()); + map.put("message", errorStatus.getReasonPhrase()); + map.put("data", Lists.newArrayList()); + + return map; + } + + private HttpStatus determineHttpStatus( + Throwable error, MergedAnnotation responseStatusAnnotation) { + if (error instanceof ResponseStatusException) { + return ((ResponseStatusException) error).getStatus(); + } + return responseStatusAnnotation + .getValue("code", HttpStatus.class) + .orElse(HttpStatus.INTERNAL_SERVER_ERROR); + } +} diff --git a/linkis-spring-cloud-services/linkis-service-gateway/linkis-spring-cloud-gateway/src/main/java/org/apache/linkis/gateway/springcloud/handler/CustomErrorWebExceptionHandler.java b/linkis-spring-cloud-services/linkis-service-gateway/linkis-spring-cloud-gateway/src/main/java/org/apache/linkis/gateway/springcloud/handler/CustomErrorWebExceptionHandler.java deleted file mode 100644 index f2bf8de354..0000000000 --- a/linkis-spring-cloud-services/linkis-service-gateway/linkis-spring-cloud-gateway/src/main/java/org/apache/linkis/gateway/springcloud/handler/CustomErrorWebExceptionHandler.java +++ /dev/null @@ -1,99 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.gateway.springcloud.handler; - -import org.springframework.boot.autoconfigure.web.ErrorProperties; -import org.springframework.boot.autoconfigure.web.ResourceProperties; -import org.springframework.boot.autoconfigure.web.reactive.error.DefaultErrorWebExceptionHandler; -import org.springframework.boot.web.error.ErrorAttributeOptions; -import org.springframework.boot.web.reactive.error.ErrorAttributes; -import org.springframework.context.ApplicationContext; -import org.springframework.core.annotation.MergedAnnotation; -import org.springframework.core.annotation.MergedAnnotations; -import org.springframework.http.HttpStatus; -import org.springframework.web.bind.annotation.ResponseStatus; -import org.springframework.web.reactive.function.server.*; -import org.springframework.web.server.ResponseStatusException; - -import java.util.LinkedHashMap; -import java.util.Map; - -import com.google.common.collect.Lists; - -public class CustomErrorWebExceptionHandler extends DefaultErrorWebExceptionHandler { - public CustomErrorWebExceptionHandler( - ErrorAttributes errorAttributes, - ResourceProperties resourceProperties, - ErrorProperties errorProperties, - ApplicationContext applicationContext) { - super(errorAttributes, resourceProperties, errorProperties, applicationContext); - } - - /** - * Get Exception Properties - * - * @param request - * @param options - */ - @Override - public Map getErrorAttributes( - ServerRequest request, ErrorAttributeOptions options) { - Map errorAttributes = new LinkedHashMap<>(); - errorAttributes.put("data", Lists.newArrayList()); - errorAttributes.put("method", request.path()); - Throwable error = getError(request); - MergedAnnotation responseStatusAnnotation = - MergedAnnotations.from(error.getClass(), MergedAnnotations.SearchStrategy.TYPE_HIERARCHY) - .get(ResponseStatus.class); - HttpStatus errorStatus = determineHttpStatus(error, responseStatusAnnotation); - errorAttributes.put("status", errorStatus.value()); - errorAttributes.put("message", errorStatus.getReasonPhrase()); - - return errorAttributes; - } - - private HttpStatus determineHttpStatus( - Throwable error, MergedAnnotation responseStatusAnnotation) { - if (error instanceof ResponseStatusException) { - return ((ResponseStatusException) error).getStatus(); - } - return responseStatusAnnotation - .getValue("code", HttpStatus.class) - .orElse(HttpStatus.INTERNAL_SERVER_ERROR); - } - - /** - * Specify the response processing method as JSON processing method - * - * @param errorAttributes - */ - @Override - public RouterFunction getRoutingFunction(ErrorAttributes errorAttributes) { - return RouterFunctions.route(RequestPredicates.all(), this::renderErrorResponse); - } - - /** - * Obtain the corresponding HttpStatus based on the status - * - * @param errorAttributes - */ - @Override - public int getHttpStatus(Map errorAttributes) { - return (int) errorAttributes.get("status"); - } -} diff --git a/linkis-spring-cloud-services/linkis-service-gateway/linkis-spring-cloud-gateway/src/main/java/org/apache/linkis/gateway/springcloud/handler/ErrorHandlerConfiguration.java b/linkis-spring-cloud-services/linkis-service-gateway/linkis-spring-cloud-gateway/src/main/java/org/apache/linkis/gateway/springcloud/handler/ErrorHandlerConfiguration.java deleted file mode 100644 index d4d47530fa..0000000000 --- a/linkis-spring-cloud-services/linkis-service-gateway/linkis-spring-cloud-gateway/src/main/java/org/apache/linkis/gateway/springcloud/handler/ErrorHandlerConfiguration.java +++ /dev/null @@ -1,81 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.gateway.springcloud.handler; - -import org.springframework.beans.factory.ObjectProvider; -import org.springframework.boot.autoconfigure.web.ResourceProperties; -import org.springframework.boot.autoconfigure.web.ServerProperties; -import org.springframework.boot.context.properties.EnableConfigurationProperties; -import org.springframework.boot.web.reactive.error.ErrorAttributes; -import org.springframework.boot.web.reactive.error.ErrorWebExceptionHandler; -import org.springframework.context.ApplicationContext; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; -import org.springframework.core.Ordered; -import org.springframework.core.annotation.Order; -import org.springframework.http.codec.ServerCodecConfigurer; -import org.springframework.web.reactive.result.view.ViewResolver; - -import java.util.Collections; -import java.util.List; - -/** - * Overwrite default exception handling - */ -@Configuration -@EnableConfigurationProperties({ServerProperties.class, ResourceProperties.class}) -public class ErrorHandlerConfiguration { - - private final ServerProperties serverProperties; - - private final ApplicationContext applicationContext; - - private final ResourceProperties resourceProperties; - - private final List viewResolvers; - - private final ServerCodecConfigurer serverCodecConfigurer; - - public ErrorHandlerConfiguration( - ServerProperties serverProperties, - ResourceProperties resourceProperties, - ObjectProvider> viewResolversProvider, - ServerCodecConfigurer serverCodecConfigurer, - ApplicationContext applicationContext) { - this.serverProperties = serverProperties; - this.applicationContext = applicationContext; - this.resourceProperties = resourceProperties; - this.viewResolvers = viewResolversProvider.getIfAvailable(Collections::emptyList); - this.serverCodecConfigurer = serverCodecConfigurer; - } - - @Bean - @Order(Ordered.HIGHEST_PRECEDENCE) - public ErrorWebExceptionHandler errorWebExceptionHandler(ErrorAttributes errorAttributes) { - CustomErrorWebExceptionHandler exceptionHandler = - new CustomErrorWebExceptionHandler( - errorAttributes, - this.resourceProperties, - this.serverProperties.getError(), - this.applicationContext); - exceptionHandler.setViewResolvers(this.viewResolvers); - exceptionHandler.setMessageWriters(this.serverCodecConfigurer.getWriters()); - exceptionHandler.setMessageReaders(this.serverCodecConfigurer.getReaders()); - return exceptionHandler; - } -} From e3d0496ba03882419e17bdaa65cfd95962e76dc7 Mon Sep 17 00:00:00 2001 From: huangKai-2323 <62878639+huangKai-2323@users.noreply.github.com> Date: Tue, 13 Jun 2023 14:55:51 +0800 Subject: [PATCH 149/261] add linkis_ps_udf_user_load index --- linkis-dist/package/db/linkis_ddl.sql | 3 ++- linkis-dist/package/db/module/linkis_udf.sql | 7 +++++-- .../upgrade/1.4.1_schema/mysql/linkis_ddl.sql | 21 +++++++++++++++++++ 3 files changed, 28 insertions(+), 3 deletions(-) create mode 100644 linkis-dist/package/db/upgrade/1.4.1_schema/mysql/linkis_ddl.sql diff --git a/linkis-dist/package/db/linkis_ddl.sql b/linkis-dist/package/db/linkis_ddl.sql index 94b94a8c32..76a1824074 100644 --- a/linkis-dist/package/db/linkis_ddl.sql +++ b/linkis-dist/package/db/linkis_ddl.sql @@ -212,7 +212,8 @@ CREATE TABLE `linkis_ps_udf_user_load` ( `user_name` varchar(50) NOT NULL, `update_time` datetime DEFAULT CURRENT_TIMESTAMP, `create_time` datetime DEFAULT CURRENT_TIMESTAMP, - PRIMARY KEY (`id`) + PRIMARY KEY (`id`), + UNIQUE KEY `uniq_uid_uname` (`udf_id`, `user_name`) ) ENGINE=InnoDB DEFAULT CHARSET=utf8; DROP TABLE IF EXISTS `linkis_ps_udf_baseinfo`; diff --git a/linkis-dist/package/db/module/linkis_udf.sql b/linkis-dist/package/db/module/linkis_udf.sql index 783591d041..3e7b2c4f13 100644 --- a/linkis-dist/package/db/module/linkis_udf.sql +++ b/linkis-dist/package/db/module/linkis_udf.sql @@ -75,9 +75,12 @@ CREATE TABLE `linkis_ps_udf_tree` ( DROP TABLE IF EXISTS `linkis_ps_udf_user_load`; CREATE TABLE `linkis_ps_udf_user_load` ( `id` bigint(20) NOT NULL AUTO_INCREMENT, - `udf_id` int(11) NOT NULL, + `udf_id` bigint(20) NOT NULL, `user_name` varchar(50) NOT NULL, - PRIMARY KEY (`id`) + `update_time` datetime DEFAULT CURRENT_TIMESTAMP, + `create_time` datetime DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY (`id`), + UNIQUE KEY `uniq_uid_uname` (`udf_id`, `user_name`) ) ENGINE=InnoDB DEFAULT CHARSET=utf8; DROP TABLE IF EXISTS `linkis_ps_udf_baseinfo`; diff --git a/linkis-dist/package/db/upgrade/1.4.1_schema/mysql/linkis_ddl.sql b/linkis-dist/package/db/upgrade/1.4.1_schema/mysql/linkis_ddl.sql new file mode 100644 index 0000000000..0147a2f346 --- /dev/null +++ b/linkis-dist/package/db/upgrade/1.4.1_schema/mysql/linkis_ddl.sql @@ -0,0 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + + +ALTER TABLE `linkis_ps_udf_user_load` ADD CONSTRAINT `uniq_uid_uname` UNIQUE (`udf_id`, `user_name`); + From 083e13169d8ab4177a7461f5fe219c683f516bee Mon Sep 17 00:00:00 2001 From: huangKai-2323 <62878639+huangKai-2323@users.noreply.github.com> Date: Thu, 18 May 2023 16:38:55 +0800 Subject: [PATCH 150/261] sync 1.4.0 code --- .../service/TaskExecutionServiceImpl.scala | 116 +++++++++--------- .../protocol/resource/ResourceProtocol.scala | 11 +- .../spark/client/context/SparkConfig.java | 13 +- ...rnApplicationClusterDescriptorAdapter.java | 2 - .../spark/SparkEngineConnPlugin.scala | 4 - 5 files changed, 71 insertions(+), 75 deletions(-) diff --git a/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/service/TaskExecutionServiceImpl.scala b/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/service/TaskExecutionServiceImpl.scala index be827a32a3..f02202eaa3 100644 --- a/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/service/TaskExecutionServiceImpl.scala +++ b/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/service/TaskExecutionServiceImpl.scala @@ -440,56 +440,18 @@ class TaskExecutionServiceImpl .isRunning(task.getStatus) ) { val progressResponse = taskProgress(task.getTaskId) - val resourceResponse: ResponseTaskYarnResource = - taskYarnResource(task.getTaskId) match { - case responseTaskYarnResource: ResponseTaskYarnResource => - if ( - responseTaskYarnResource.resourceMap != null && !responseTaskYarnResource.resourceMap.isEmpty - ) { - responseTaskYarnResource - } else { - null - } - case _ => - null - } - val extraInfoMap = new util.HashMap[String, Object]() - extraInfoMap.put(TaskConstant.ENGINE_INSTANCE, Sender.getThisInstance) - extraInfoMap.put( - ECConstants.EC_TICKET_ID_KEY, - EngineConnObject.getEngineCreationContext.getTicketId - ) - val ecParams = EngineConnObject.getEngineCreationContext.getOptions - if (ecParams.containsKey(ECConstants.YARN_QUEUE_NAME_CONFIG_KEY)) { - extraInfoMap.put( - ECConstants.YARN_QUEUE_NAME_KEY, - ecParams.get(ECConstants.YARN_QUEUE_NAME_CONFIG_KEY) - ) - } - extraInfoMap.put(TaskConstant.ENGINE_CONN_TASK_ID, task.getTaskId) - extraInfoMap.put( - TaskConstant.ENGINE_CONN_SUBMIT_TIME, - System.currentTimeMillis.toString - ) + val resourceResponse = buildResourceMap(task) + val extraInfoMap = buildExtraInfoMap(task) // todo add other info - var respRunningInfo: ResponseTaskRunningInfo = null - if (null != resourceResponse) { - respRunningInfo = ResponseTaskRunningInfo( - progressResponse.execId, - progressResponse.progress, - progressResponse.progressInfo, - resourceResponse.resourceMap, - extraInfoMap - ) - } else { - respRunningInfo = ResponseTaskRunningInfo( - progressResponse.execId, - progressResponse.progress, - progressResponse.progressInfo, - null, - extraInfoMap - ) - } + val resourceMap = if (null != resourceResponse) resourceResponse.resourceMap else null + + val respRunningInfo: ResponseTaskRunningInfo = ResponseTaskRunningInfo( + progressResponse.execId, + progressResponse.progress, + progressResponse.progressInfo, + resourceMap, + extraInfoMap + ) sendToEntrance(task, respRunningInfo) Thread.sleep(TimeUnit.MILLISECONDS.convert(sleepInterval, TimeUnit.SECONDS)) } @@ -498,6 +460,42 @@ class TaskExecutionServiceImpl }) } + private def buildExtraInfoMap(task: EngineConnTask): util.HashMap[String, Object] = { + val extraInfoMap = new util.HashMap[String, Object]() + extraInfoMap.put(TaskConstant.ENGINE_INSTANCE, Sender.getThisInstance) + extraInfoMap.put( + ECConstants.EC_TICKET_ID_KEY, + EngineConnObject.getEngineCreationContext.getTicketId + ) + val ecParams = EngineConnObject.getEngineCreationContext.getOptions + if (ecParams.containsKey(ECConstants.YARN_QUEUE_NAME_CONFIG_KEY)) { + extraInfoMap.put( + ECConstants.YARN_QUEUE_NAME_KEY, + ecParams.get(ECConstants.YARN_QUEUE_NAME_CONFIG_KEY) + ) + } + extraInfoMap.put(TaskConstant.ENGINE_CONN_TASK_ID, task.getTaskId) + extraInfoMap.put(TaskConstant.ENGINE_CONN_SUBMIT_TIME, System.currentTimeMillis.toString) + extraInfoMap + } + + private def buildResourceMap(task: EngineConnTask): ResponseTaskYarnResource = { + val resourceResponse: ResponseTaskYarnResource = + taskYarnResource(task.getTaskId) match { + case responseTaskYarnResource: ResponseTaskYarnResource => + if ( + responseTaskYarnResource.resourceMap != null && !responseTaskYarnResource.resourceMap.isEmpty + ) { + responseTaskYarnResource + } else { + null + } + case _ => + null + } + resourceResponse + } + private def taskYarnResource(taskID: String): ResponseTaskYarnResource = { val executor = taskIdCache.getIfPresent(taskID) executor match { @@ -677,14 +675,20 @@ class TaskExecutionServiceImpl if (EngineConnConf.ENGINE_PUSH_LOG_TO_ENTRANCE.getValue) { val task = getTaskByTaskId(taskProgressUpdateEvent.taskId) if (null != task) { - sendToEntrance( - task, - ResponseTaskProgress( - taskProgressUpdateEvent.taskId, - taskProgressUpdateEvent.progress, - taskProgressUpdateEvent.progressInfo - ) + val resourceResponse = buildResourceMap(task) + val extraInfoMap = buildExtraInfoMap(task) + + val resourceMap = if (null != resourceResponse) resourceResponse.resourceMap else null + + val respRunningInfo: ResponseTaskRunningInfo = ResponseTaskRunningInfo( + taskProgressUpdateEvent.taskId, + taskProgressUpdateEvent.progress, + taskProgressUpdateEvent.progressInfo, + resourceMap, + extraInfoMap ) + + sendToEntrance(task, respRunningInfo) } else { logger.error( "Task cannot null! taskProgressUpdateEvent : " + ComputationEngineUtils.GSON diff --git a/linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/scala/org/apache/linkis/manager/common/protocol/resource/ResourceProtocol.scala b/linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/scala/org/apache/linkis/manager/common/protocol/resource/ResourceProtocol.scala index 20fa958e7d..c22f8824c2 100644 --- a/linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/scala/org/apache/linkis/manager/common/protocol/resource/ResourceProtocol.scala +++ b/linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/scala/org/apache/linkis/manager/common/protocol/resource/ResourceProtocol.scala @@ -45,4 +45,13 @@ case class ResponseTaskRunningInfo( resourceMap: util.HashMap[String, ResourceWithStatus], extraInfoMap: util.HashMap[String, Object] ) extends RetryableProtocol - with RequestProtocol + with RequestProtocol { + + private val mutableResourceMap = Option(resourceMap).getOrElse(new util.HashMap) + private val mutableExtraInfoMap = Option(extraInfoMap).getOrElse(new util.HashMap) + + def getResourceMaps: util.HashMap[String, ResourceWithStatus] = mutableResourceMap + + def getExtraInfoMap: util.HashMap[String, Object] = mutableExtraInfoMap + +} diff --git a/linkis-engineconn-plugins/spark/src/main/java/org/apache/linkis/engineplugin/spark/client/context/SparkConfig.java b/linkis-engineconn-plugins/spark/src/main/java/org/apache/linkis/engineplugin/spark/client/context/SparkConfig.java index c8f7842220..20b0749d83 100644 --- a/linkis-engineconn-plugins/spark/src/main/java/org/apache/linkis/engineplugin/spark/client/context/SparkConfig.java +++ b/linkis-engineconn-plugins/spark/src/main/java/org/apache/linkis/engineplugin/spark/client/context/SparkConfig.java @@ -27,7 +27,6 @@ public class SparkConfig { private String master = "yarn"; // ("yarn") private String deployMode = "client"; // ("client") // todo cluster private String appResource; // ("") - // private String mainClass; // ("") private String appName; // ("") private String jars; // ("--jars", "") private String packages; // ("--packages", "") @@ -92,14 +91,6 @@ public void setAppResource(String appResource) { this.appResource = appResource; } - // public String getMainClass() { - // return mainClass; - // } - // - // public void setMainClass(String mainClass) { - // this.mainClass = mainClass; - // } - public String getAppName() { return appName; } @@ -299,9 +290,7 @@ public String toString() { + ", deployMode='" + deployMode + '\'' - + - // ", mainClass='" + mainClass + '\'' + - ", appName='" + + ", appName='" + appName + '\'' + ", jars='" diff --git a/linkis-engineconn-plugins/spark/src/main/java/org/apache/linkis/engineplugin/spark/client/deployment/YarnApplicationClusterDescriptorAdapter.java b/linkis-engineconn-plugins/spark/src/main/java/org/apache/linkis/engineplugin/spark/client/deployment/YarnApplicationClusterDescriptorAdapter.java index 21f2885b4d..9c753d862f 100644 --- a/linkis-engineconn-plugins/spark/src/main/java/org/apache/linkis/engineplugin/spark/client/deployment/YarnApplicationClusterDescriptorAdapter.java +++ b/linkis-engineconn-plugins/spark/src/main/java/org/apache/linkis/engineplugin/spark/client/deployment/YarnApplicationClusterDescriptorAdapter.java @@ -47,7 +47,6 @@ public void deployCluster(String mainClass, String args, Map con .setMaster(sparkConfig.getMaster()) .setDeployMode(sparkConfig.getDeployMode()) .setAppName(sparkConfig.getAppName()) - // .setPropertiesFile("") .setVerbose(true); sparkLauncher.setConf("spark.app.name", sparkConfig.getAppName()); if (confMap != null) confMap.forEach((k, v) -> sparkLauncher.setConf(k, v)); @@ -75,7 +74,6 @@ public void deployCluster(String mainClass, String args, Map con Arrays.stream(args.split("\\s+")) .filter(StringUtils::isNotBlank) .forEach(arg -> sparkLauncher.addAppArgs(arg)); - // sparkLauncher.addAppArgs(args); sparkAppHandle = sparkLauncher.startApplication( new SparkAppHandle.Listener() { diff --git a/linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/SparkEngineConnPlugin.scala b/linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/SparkEngineConnPlugin.scala index ea8c61eed0..c1cd0a4792 100644 --- a/linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/SparkEngineConnPlugin.scala +++ b/linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/SparkEngineConnPlugin.scala @@ -38,14 +38,10 @@ class SparkEngineConnPlugin extends EngineConnPlugin { private val resourceLocker = new Object() - private val engineLaunchBuilderLocker = new Object() - private val engineFactoryLocker = new Object() private var engineResourceFactory: EngineResourceFactory = _ - private var engineLaunchBuilder: EngineConnLaunchBuilder = _ - private var engineFactory: EngineConnFactory = _ override def init(params: util.Map[String, AnyRef]): Unit = { From 4bb615ca6d2bb84b812819df6fa2e0ea7b4721bf Mon Sep 17 00:00:00 2001 From: huangKai-2323 <62878639+huangKai-2323@users.noreply.github.com> Date: Wed, 14 Jun 2023 10:08:53 +0800 Subject: [PATCH 151/261] fix spark memory count error --- .../engineplugin/spark/executor/SparkEngineConnExecutor.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/executor/SparkEngineConnExecutor.scala b/linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/executor/SparkEngineConnExecutor.scala index 04e735d84e..47275e2ba9 100644 --- a/linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/executor/SparkEngineConnExecutor.scala +++ b/linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/executor/SparkEngineConnExecutor.scala @@ -139,9 +139,9 @@ abstract class SparkEngineConnExecutor(val sc: SparkContext, id: Long) Utils.tryCatch({ val executorNum: Int = sc.getConf.get("spark.executor.instances").toInt val executorMem: Long = - ByteTimeUtils.byteStringAsGb(sc.getConf.get("spark.executor.memory")) * executorNum + ByteTimeUtils.byteStringAsGb(sc.getConf.get("spark.executor.memory")) val driverMem: Long = ByteTimeUtils.byteStringAsGb(sc.getConf.get("spark.driver.memory")) - val sparkExecutorCores = sc.getConf.get("spark.executor.cores", "2").toInt * executorNum + val sparkExecutorCores = sc.getConf.get("spark.executor.cores", "2").toInt val sparkDriverCores = sc.getConf.get("spark.driver.cores", "1").toInt val queue = sc.getConf.get("spark.yarn.queue") // with unit if set configuration with unit From 9f8c44e27866a58fad9e0d8bf4fd0138cc32f16e Mon Sep 17 00:00:00 2001 From: ahaoyao <731753729@qq.com> Date: Fri, 26 May 2023 15:47:19 +0800 Subject: [PATCH 152/261] Security Work Order Upgrade --- linkis-commons/linkis-module/pom.xml | 1 + linkis-engineconn-plugins/sqoop/pom.xml | 2 +- tool/dependencies/known-dependencies.txt | 3 ++- 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/linkis-commons/linkis-module/pom.xml b/linkis-commons/linkis-module/pom.xml index b70331e0d1..cbaf0295fd 100644 --- a/linkis-commons/linkis-module/pom.xml +++ b/linkis-commons/linkis-module/pom.xml @@ -294,6 +294,7 @@ org.springframework.security spring-security-crypto + 5.7.5 diff --git a/linkis-engineconn-plugins/sqoop/pom.xml b/linkis-engineconn-plugins/sqoop/pom.xml index 1753d21654..8435ca1705 100644 --- a/linkis-engineconn-plugins/sqoop/pom.xml +++ b/linkis-engineconn-plugins/sqoop/pom.xml @@ -63,7 +63,7 @@ org.apache.avro avro - 1.10.2 + 1.11.0 provided diff --git a/tool/dependencies/known-dependencies.txt b/tool/dependencies/known-dependencies.txt index 62f2e364a2..c1aad5a9b5 100644 --- a/tool/dependencies/known-dependencies.txt +++ b/tool/dependencies/known-dependencies.txt @@ -32,6 +32,7 @@ avatica-1.8.0.jar avatica-metrics-1.8.0.jar avro-1.7.4.jar avro-1.7.7.jar +avro-1.11.0.jar bcpkix-jdk15on-1.64.jar bcprov-jdk15on-1.64.jar bonecp-0.8.0.RELEASE.jar @@ -522,7 +523,7 @@ spring-jcl-5.2.22.RELEASE.jar spring-jdbc-5.2.22.RELEASE.jar spring-plugin-core-2.0.0.RELEASE.jar spring-plugin-metadata-2.0.0.RELEASE.jar -spring-security-crypto-5.3.9.RELEASE.jar +spring-security-crypto-5.7.5.RELEASE.jar spring-security-rsa-1.0.9.RELEASE.jar spring-tx-5.2.22.RELEASE.jar spring-web-5.2.22.RELEASE.jar From 9f6594bb665cf95024e585aba69f19fb52deb53c Mon Sep 17 00:00:00 2001 From: huangKai-2323 <62878639+huangKai-2323@users.noreply.github.com> Date: Thu, 18 May 2023 11:32:40 +0800 Subject: [PATCH 153/261] udf all add description --- .../org/apache/linkis/udf/entity/UDFInfo.java | 10 ++++++++ .../apache/linkis/udf/api/UDFRestfulApi.java | 3 ++- .../main/resources/mapper/common/UDFDao.xml | 18 ++++++++++----- .../src/test/resources/create.sql | 23 +++++++++++-------- 4 files changed, 38 insertions(+), 16 deletions(-) diff --git a/linkis-public-enhancements/linkis-udf/linkis-udf-common/src/main/java/org/apache/linkis/udf/entity/UDFInfo.java b/linkis-public-enhancements/linkis-udf/linkis-udf-common/src/main/java/org/apache/linkis/udf/entity/UDFInfo.java index 502047ac6a..19d1793608 100644 --- a/linkis-public-enhancements/linkis-udf/linkis-udf-common/src/main/java/org/apache/linkis/udf/entity/UDFInfo.java +++ b/linkis-public-enhancements/linkis-udf/linkis-udf-common/src/main/java/org/apache/linkis/udf/entity/UDFInfo.java @@ -33,6 +33,7 @@ public class UDFInfo { private String clusterName; private Boolean isLoad; + private String description; public UDFInfo() {}; @@ -61,6 +62,7 @@ public UDFInfo( this.createTime = createTime; this.updateTime = updateTime; this.isLoad = isLoad; + this.description = description; } public Long getId() { @@ -158,4 +160,12 @@ public String getClusterName() { public void setClusterName(String clusterName) { this.clusterName = clusterName; } + + public String getDescription() { + return description; + } + + public void setDescription(String description) { + this.description = description; + } } diff --git a/linkis-public-enhancements/linkis-udf/linkis-udf-service/src/main/java/org/apache/linkis/udf/api/UDFRestfulApi.java b/linkis-public-enhancements/linkis-udf/linkis-udf-service/src/main/java/org/apache/linkis/udf/api/UDFRestfulApi.java index c659e4aa93..552e3a8c22 100644 --- a/linkis-public-enhancements/linkis-udf/linkis-udf-service/src/main/java/org/apache/linkis/udf/api/UDFRestfulApi.java +++ b/linkis-public-enhancements/linkis-udf/linkis-udf-service/src/main/java/org/apache/linkis/udf/api/UDFRestfulApi.java @@ -88,7 +88,8 @@ public class UDFRestfulApi { public Message allUDF(HttpServletRequest req, String jsonString) { Message message = null; try { - String userName = ModuleUserUtils.getOperationUser(req, "get all udfs "); + // String userName = ModuleUserUtils.getOperationUser(req, "get all udfs "); + String userName = "hadoop"; if (!StringUtils.isEmpty(jsonString)) { Map json = mapper.reader(Map.class).readValue(jsonString); String type = (String) json.getOrDefault("type", "self"); diff --git a/linkis-public-enhancements/linkis-udf/linkis-udf-service/src/main/resources/mapper/common/UDFDao.xml b/linkis-public-enhancements/linkis-udf/linkis-udf-service/src/main/resources/mapper/common/UDFDao.xml index c876e92342..020960aa25 100644 --- a/linkis-public-enhancements/linkis-udf/linkis-udf-service/src/main/resources/mapper/common/UDFDao.xml +++ b/linkis-public-enhancements/linkis-udf/linkis-udf-service/src/main/resources/mapper/common/UDFDao.xml @@ -31,6 +31,7 @@ + @@ -114,9 +115,13 @@ select - - from linkis_ps_udf_baseinfo - where id in + a.id, a.`create_user`, a.`udf_name`, a.`udf_type`, + a.`is_expire`,a.`is_shared`,a.`tree_id`,a.`create_time`,a.`update_time`,a.`sys`,a.`cluster_name`,b.description + from linkis_ps_udf_baseinfo a , (select a.* from linkis_ps_udf_version a ,(SELECT udf_id ,MAX(bml_resource_version) as bml_resource_version FROM linkis_ps_udf_version GROUP BY udf_id) b where b.udf_id=a.udf_id and b.bml_resource_version = a.bml_resource_version) b + where a.id = b.udf_id and a.id in (select udf_id from linkis_ps_udf_shared_info where user_name=#{userName}) diff --git a/linkis-public-enhancements/linkis-udf/linkis-udf-service/src/test/resources/create.sql b/linkis-public-enhancements/linkis-udf/linkis-udf-service/src/test/resources/create.sql index c22d27b529..f8c41badc7 100644 --- a/linkis-public-enhancements/linkis-udf/linkis-udf-service/src/test/resources/create.sql +++ b/linkis-public-enhancements/linkis-udf/linkis-udf-service/src/test/resources/create.sql @@ -20,11 +20,13 @@ SET REFERENTIAL_INTEGRITY FALSE; DROP TABLE IF EXISTS linkis_ps_udf_user_load CASCADE; CREATE TABLE IF NOT EXISTS linkis_ps_udf_user_load ( - id bigint(20) NOT NULL AUTO_INCREMENT, - udf_id bigint(20) NOT NULL, - user_name varchar(50) NOT NULL, - PRIMARY KEY (id) -) ; + `id` bigint(20) NOT NULL AUTO_INCREMENT, + `udf_id` bigint(20) NOT NULL, + `user_name` varchar(50) NOT NULL, + `update_time` datetime DEFAULT CURRENT_TIMESTAMP, + `create_time` datetime DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY (`id`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8; DROP TABLE IF EXISTS linkis_ps_udf_baseinfo CASCADE; CREATE TABLE IF NOT EXISTS linkis_ps_udf_baseinfo ( @@ -67,6 +69,7 @@ CREATE TABLE IF NOT EXISTS linkis_ps_udf_version ( use_format varchar(255) DEFAULT NULL, description varchar(255) NOT NULL COMMENT 'version desc', create_time timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + update_time timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, md5 varchar(100) DEFAULT NULL, PRIMARY KEY (id) ) ; @@ -82,7 +85,9 @@ CREATE TABLE IF NOT EXISTS linkis_ps_udf_shared_info ( DROP TABLE IF EXISTS linkis_ps_udf_manager CASCADE; CREATE TABLE IF NOT EXISTS linkis_ps_udf_manager ( - id bigint(20) NOT NULL AUTO_INCREMENT, - user_name varchar(20) DEFAULT NULL, - PRIMARY KEY (id) -) ; \ No newline at end of file + `id` bigint(20) NOT NULL AUTO_INCREMENT, + `user_name` varchar(20) DEFAULT NULL, + `update_time` datetime DEFAULT CURRENT_TIMESTAMP, + `create_time` datetime DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY (`id`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8; \ No newline at end of file From e43df3594cb7c25e9da186e4e64b8b51d64de79f Mon Sep 17 00:00:00 2001 From: huangKai-2323 <62878639+huangKai-2323@users.noreply.github.com> Date: Mon, 29 May 2023 17:14:11 +0800 Subject: [PATCH 154/261] Code optimization --- .../src/main/java/org/apache/linkis/udf/api/UDFRestfulApi.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/linkis-public-enhancements/linkis-udf/linkis-udf-service/src/main/java/org/apache/linkis/udf/api/UDFRestfulApi.java b/linkis-public-enhancements/linkis-udf/linkis-udf-service/src/main/java/org/apache/linkis/udf/api/UDFRestfulApi.java index 552e3a8c22..c659e4aa93 100644 --- a/linkis-public-enhancements/linkis-udf/linkis-udf-service/src/main/java/org/apache/linkis/udf/api/UDFRestfulApi.java +++ b/linkis-public-enhancements/linkis-udf/linkis-udf-service/src/main/java/org/apache/linkis/udf/api/UDFRestfulApi.java @@ -88,8 +88,7 @@ public class UDFRestfulApi { public Message allUDF(HttpServletRequest req, String jsonString) { Message message = null; try { - // String userName = ModuleUserUtils.getOperationUser(req, "get all udfs "); - String userName = "hadoop"; + String userName = ModuleUserUtils.getOperationUser(req, "get all udfs "); if (!StringUtils.isEmpty(jsonString)) { Map json = mapper.reader(Map.class).readValue(jsonString); String type = (String) json.getOrDefault("type", "self"); From be19cfd79865d03d5e680423ef999386d51cf82b Mon Sep 17 00:00:00 2001 From: huangKai-2323 <62878639+huangKai-2323@users.noreply.github.com> Date: Mon, 12 Jun 2023 14:33:55 +0800 Subject: [PATCH 155/261] Code optimization --- .../main/resources/mapper/common/UDFDao.xml | 31 ++++++++++--------- 1 file changed, 17 insertions(+), 14 deletions(-) diff --git a/linkis-public-enhancements/linkis-udf/linkis-udf-service/src/main/resources/mapper/common/UDFDao.xml b/linkis-public-enhancements/linkis-udf/linkis-udf-service/src/main/resources/mapper/common/UDFDao.xml index 020960aa25..6673074780 100644 --- a/linkis-public-enhancements/linkis-udf/linkis-udf-service/src/main/resources/mapper/common/UDFDao.xml +++ b/linkis-public-enhancements/linkis-udf/linkis-udf-service/src/main/resources/mapper/common/UDFDao.xml @@ -114,26 +114,29 @@ SELECT - info.id, info.`create_user`, info.`udf_name`, info.`udf_type`, - info.`is_expire`,info.`is_shared`,info.`tree_id`,info.`create_time`,info.`update_time`,info.`sys`,info.`cluster_name`,udf_version.description - FROM linkis_ps_udf_baseinfo info , - (SELECT udf_version.* FROM linkis_ps_udf_version udf_version , - (SELECT udf_id ,MAX(bml_resource_version) AS bml_resource_version FROM linkis_ps_udf_version GROUP BY udf_id) version_tmp - WHERE version_tmp.udf_id=udf_version.udf_id AND version_tmp.bml_resource_version = udf_version.bml_resource_version) udf_version - WHERE info.id = udf_version.udf_id AND info.create_user in + info.id,info.`create_user`,info.`udf_name`,info.`udf_type`,info.`is_expire`,info.`is_shared`,info.`tree_id`,info.`create_time`,info.`update_time`, + info.`sys`,info.`cluster_name`,udf_version.description + FROM + linkis_ps_udf_baseinfo info , + (SELECT + udf_version.* + FROM + linkis_ps_udf_version udf_version , ( + SELECT + udf_id , MAX(bml_resource_version) AS bml_resource_version + FROM + linkis_ps_udf_version + GROUP BY + udf_id) version_tmp + WHERE + version_tmp.udf_id = udf_version.udf_id + AND version_tmp.bml_resource_version = udf_version.bml_resource_version) udf_version + WHERE + info.id = udf_version.udf_id + AND info.create_user in #{item} @@ -129,14 +142,29 @@ From de66da973b594bcb993691b0ac087389737d862f Mon Sep 17 00:00:00 2001 From: v-kkhuang <62878639+v-kkhuang@users.noreply.github.com> Date: Sun, 18 Jun 2023 18:43:06 +0800 Subject: [PATCH 161/261] =?UTF-8?q?=E3=80=901.1.13=E3=80=91interface=20fil?= =?UTF-8?q?einfo=20add=20totalline=20(#165)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fileInfo add totalLine --------- Co-authored-by: huangKai-2323 <62878639+huangKai-2323@users.noreply.github.com> --- .../storage/utils/FileSystemUtils.scala | 20 +++++++++++++++++++ .../filesystem/restful/api/FsRestfulApi.java | 8 ++++++++ 2 files changed, 28 insertions(+) diff --git a/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/utils/FileSystemUtils.scala b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/utils/FileSystemUtils.scala index 5252c12e03..68296401d3 100644 --- a/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/utils/FileSystemUtils.scala +++ b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/utils/FileSystemUtils.scala @@ -22,6 +22,10 @@ import org.apache.linkis.common.utils.{Logging, Utils} import org.apache.linkis.storage.FSFactory import org.apache.linkis.storage.fs.FileSystem import org.apache.linkis.storage.fs.impl.LocalFileSystem +import org.apache.linkis.storage.source.{FileSource, FileSource$} + +import org.apache.commons.io.IOUtils +import org.apache.commons.math3.util.Pair import java.io.IOException import java.util @@ -113,4 +117,20 @@ object FileSystemUtils extends Logging { true } + /** + * Return the total number of lines in the file + * + * @param fsPath + * @param fileSystem + * @return + * TotalLine + */ + def getTotalLine(fsPath: FsPath, fileSystem: FileSystem): Int = { + val fileSource = FileSource.create(fsPath, fileSystem) + try { + fileSource.collect() + fileSource.getTotalLine + } finally IOUtils.closeQuietly(fileSource) + } + } diff --git a/linkis-public-enhancements/linkis-script-dev/linkis-storage-script-dev-server/src/main/java/org/apache/linkis/filesystem/restful/api/FsRestfulApi.java b/linkis-public-enhancements/linkis-script-dev/linkis-storage-script-dev-server/src/main/java/org/apache/linkis/filesystem/restful/api/FsRestfulApi.java index 873f239032..6e38e52bbd 100644 --- a/linkis-public-enhancements/linkis-script-dev/linkis-storage-script-dev-server/src/main/java/org/apache/linkis/filesystem/restful/api/FsRestfulApi.java +++ b/linkis-public-enhancements/linkis-script-dev/linkis-storage-script-dev-server/src/main/java/org/apache/linkis/filesystem/restful/api/FsRestfulApi.java @@ -41,6 +41,7 @@ import org.apache.linkis.storage.script.*; import org.apache.linkis.storage.source.FileSource; import org.apache.linkis.storage.source.FileSource$; +import org.apache.linkis.storage.utils.FileSystemUtils; import org.apache.linkis.storage.utils.StorageUtils; import org.apache.commons.io.IOUtils; @@ -524,13 +525,20 @@ public Message fileInfo( Pair[] fileInfo = fileSource.getFileInfo(pageSize); IOUtils.closeQuietly(fileSource); if (null != fileInfo && fileInfo.length > 0) { + int rowNumber = (int) fileInfo[0].getSecond(); message.data("path", path); message.data("colNumber", fileInfo[0].getFirst()); message.data("rowNumber", fileInfo[0].getSecond()); + if (rowNumber >= pageSize) { + message.data("totalLine", FileSystemUtils.getTotalLine(fsPath, fileSystem)); + } else { + message.data("totalLine", rowNumber); + } } else { message.data("path", path); message.data("colNumber", 0); message.data("rowNumber", 0); + message.data("totalLine", 0); } return message; } finally { From 1f6caf47dfbfa21aa30977cdc45e58e295aeeaa0 Mon Sep 17 00:00:00 2001 From: v-kkhuang <62878639+v-kkhuang@users.noreply.github.com> Date: Sun, 18 Jun 2023 19:20:44 +0800 Subject: [PATCH 162/261] =?UTF-8?q?=E3=80=901.1.13=E3=80=91remove=20tenant?= =?UTF-8?q?=20for=20entrance=20(#163)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Code optimization for AskEngineConnHook --------- Co-authored-by: huangKai-2323 <62878639+huangKai-2323@users.noreply.github.com> --- .../conf/EntranceSpringConfiguration.java | 2 +- ...ngineHook.scala => AskEngineConnHook.scala} | 4 ++-- ...xt.scala => AskEngineConnHookContext.scala} | 2 +- .../engine/DefaultEngineAskEngineService.scala | 18 +++++++++--------- 4 files changed, 13 insertions(+), 13 deletions(-) rename linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/hook/{AskEngineHook.scala => AskEngineConnHook.scala} (92%) rename linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/hook/{AskEngineHookContext.scala => AskEngineConnHookContext.scala} (93%) diff --git a/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/conf/EntranceSpringConfiguration.java b/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/conf/EntranceSpringConfiguration.java index cf520c3823..86b1a91f7a 100644 --- a/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/conf/EntranceSpringConfiguration.java +++ b/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/conf/EntranceSpringConfiguration.java @@ -150,7 +150,7 @@ public EntranceInterceptor[] entranceInterceptors() { new ScalaCodeInterceptor(), new SQLLimitEntranceInterceptor(), new CommentInterceptor(), - new SetTenantLabelInterceptor(), + // new SetTenantLabelInterceptor(), new UserCreatorIPCheckInterceptor() }; } diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/hook/AskEngineHook.scala b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/hook/AskEngineConnHook.scala similarity index 92% rename from linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/hook/AskEngineHook.scala rename to linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/hook/AskEngineConnHook.scala index 03bb7126aa..7240288698 100644 --- a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/hook/AskEngineHook.scala +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/hook/AskEngineConnHook.scala @@ -17,9 +17,9 @@ package org.apache.linkis.manager.am.hook -trait AskEngineHook { +trait AskEngineConnHook { @throws(classOf[Exception]) - def doHook(ctx: AskEngineHookContext): Unit + def doHook(ctx: AskEngineConnHookContext): Unit } diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/hook/AskEngineHookContext.scala b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/hook/AskEngineConnHookContext.scala similarity index 93% rename from linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/hook/AskEngineHookContext.scala rename to linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/hook/AskEngineConnHookContext.scala index 38aa1789b0..d5fcdb9686 100644 --- a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/hook/AskEngineHookContext.scala +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/hook/AskEngineConnHookContext.scala @@ -20,7 +20,7 @@ package org.apache.linkis.manager.am.hook import org.apache.linkis.manager.common.protocol.engine.EngineAskRequest import org.apache.linkis.rpc.Sender -class AskEngineHookContext(request: EngineAskRequest, sender: Sender) { +class AskEngineConnHookContext(request: EngineAskRequest, sender: Sender) { def getRequest(): EngineAskRequest = request def getSender(): Sender = sender } diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/service/engine/DefaultEngineAskEngineService.scala b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/service/engine/DefaultEngineAskEngineService.scala index 94754e9cd8..e0d35bea12 100644 --- a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/service/engine/DefaultEngineAskEngineService.scala +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/service/engine/DefaultEngineAskEngineService.scala @@ -21,7 +21,7 @@ import org.apache.linkis.common.exception.LinkisRetryException import org.apache.linkis.common.utils.{Logging, Utils} import org.apache.linkis.governance.common.utils.{JobUtils, LoggerUtils} import org.apache.linkis.manager.am.conf.AMConfiguration -import org.apache.linkis.manager.am.hook.{AskEngineHook, AskEngineHookContext} +import org.apache.linkis.manager.am.hook.{AskEngineConnHook, AskEngineConnHookContext} import org.apache.linkis.manager.common.constant.AMConstant import org.apache.linkis.manager.common.protocol.engine._ import org.apache.linkis.manager.label.constant.LabelKeyConstant @@ -58,8 +58,8 @@ class DefaultEngineAskEngineService @Autowired(required = false) @Qualifier - /* hook的实现类必须加上@Qualifier注解才能生效 */ - var hooks: Array[AskEngineHook] = _ + /* The implementation class of hook must be annotated with @Qualifier to take effect(hook的实现类必须加上@Qualifier注解才能生效) */ + var hooksArray: Array[AskEngineConnHook] = _ private val idCreator = new AtomicInteger() @@ -74,11 +74,11 @@ class DefaultEngineAskEngineService @Receiver override def askEngine(engineAskRequest: EngineAskRequest, sender: Sender): Any = { - if (hooks != null && hooks.size > 0) { - val ctx = new AskEngineHookContext(engineAskRequest, sender) + if (hooksArray != null && hooksArray.size > 0) { + val ctx = new AskEngineConnHookContext(engineAskRequest, sender) - /** hook中抛异常会阻断 */ - hooks.foreach(h => + /** Throwing exceptions in hook will block(hook中抛异常会阻断) */ + hooksArray.foreach(h => Utils.tryCatch(h.doHook(ctx)) { t => { val engineAskAsyncId = getAsyncId @@ -139,7 +139,7 @@ class DefaultEngineAskEngineService logger.info( s"Task: $taskId start to async($engineAskAsyncId) createEngine, ${engineAskRequest.getCreateService}" ) - // 如果原来的labels含engineInstance ，先去掉 + // If the original labels contain engineInstance, remove it first (如果原来的labels含engineInstance ，先去掉) engineAskRequest.getLabels.remove("engineInstance") val engineCreateRequest = new EngineCreateRequest engineCreateRequest.setLabels(engineAskRequest.getLabels) @@ -153,7 +153,7 @@ class DefaultEngineAskEngineService if (engineCreateRequest.getTimeout <= 0) { AMConfiguration.ENGINE_START_MAX_TIME.getValue.toLong } else engineCreateRequest.getTimeout - // useEngine 需要加上超时 + // UseEngine requires a timeout (useEngine 需要加上超时) val createEngineNode = getEngineNodeManager.useEngine(createNode, timeout) if (null == createEngineNode) { throw new LinkisRetryException( From 424ee85952c455140b846fa547b5a9423db04383 Mon Sep 17 00:00:00 2001 From: v-kkhuang <62878639+v-kkhuang@users.noreply.github.com> Date: Sun, 18 Jun 2023 19:22:01 +0800 Subject: [PATCH 163/261] =?UTF-8?q?=E3=80=901.1.13=E3=80=91Adding=20specif?= =?UTF-8?q?ied=20parameter=20validation=20when=20modifying=20configuration?= =?UTF-8?q?=20files=20(#180)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * config add value intercept for ConfigurationRestfulApi saveKeyValue --------- Co-authored-by: huangKai-2323 <62878639+huangKai-2323@users.noreply.github.com> Co-authored-by: casionone --- .../configuration/restful/api/ConfigurationRestfulApi.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/linkis-public-enhancements/linkis-configuration/src/main/java/org/apache/linkis/configuration/restful/api/ConfigurationRestfulApi.java b/linkis-public-enhancements/linkis-configuration/src/main/java/org/apache/linkis/configuration/restful/api/ConfigurationRestfulApi.java index 283960d5df..8dad9e3f36 100644 --- a/linkis-public-enhancements/linkis-configuration/src/main/java/org/apache/linkis/configuration/restful/api/ConfigurationRestfulApi.java +++ b/linkis-public-enhancements/linkis-configuration/src/main/java/org/apache/linkis/configuration/restful/api/ConfigurationRestfulApi.java @@ -443,7 +443,7 @@ public Message saveKeyValue(HttpServletRequest req, @RequestBody Map Date: Sun, 18 Jun 2023 19:24:19 +0800 Subject: [PATCH 164/261] =?UTF-8?q?=E3=80=901.1.13=E3=80=91=20Operation=20?= =?UTF-8?q?and=20maintenance=20optimization:=20Service=20version=20informa?= =?UTF-8?q?tion=20reporting=20(#182)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * service metadata version information reporting --------- Co-authored-by: huangKai-2323 <62878639+huangKai-2323@users.noreply.github.com> Co-authored-by: casionone --- .../linkis/common/conf/BDPConfiguration.scala | 29 +++++++++++++++++++ .../server/utils/LinkisMainHelper.scala | 4 ++- linkis-dist/package/conf/version.properties | 5 ++++ 3 files changed, 37 insertions(+), 1 deletion(-) create mode 100644 linkis-dist/package/conf/version.properties diff --git a/linkis-commons/linkis-common/src/main/scala/org/apache/linkis/common/conf/BDPConfiguration.scala b/linkis-commons/linkis-common/src/main/scala/org/apache/linkis/common/conf/BDPConfiguration.scala index 14febab63a..b97b022a00 100644 --- a/linkis-commons/linkis-common/src/main/scala/org/apache/linkis/common/conf/BDPConfiguration.scala +++ b/linkis-commons/linkis-common/src/main/scala/org/apache/linkis/common/conf/BDPConfiguration.scala @@ -36,6 +36,8 @@ private[conf] object BDPConfiguration extends Logging { val DEFAULT_SERVER_CONF_FILE_NAME = "linkis-server.properties" + val DEFAULT_VERSION_FILE_NAME = "version.properties" + val DEFAULT_CONFIG_HOT_LOAD_DELAY_MILLS = 3 * 60 * 1000L private val extractConfig = new Properties @@ -98,7 +100,34 @@ private[conf] object BDPConfiguration extends Logging { } } } + // load version conf + val versionConf = sysProps.getOrElse("linkis.version.conf", DEFAULT_VERSION_FILE_NAME) + + + // version conf file path(env LINKIS_VERSION_CONF_FILE_PATH > classpath) + var versionConfPath = sysProps.getOrElse("LINKIS_VERSION_CONF_FILE_PATH", "") + if(StringUtils.isBlank(versionConfPath)) { + logger.info( + s"LINKIS_VERSION_CONF_FILE_PATH is empty, try to use version.properties file path from classpath" + ) + val versionConfFileURL = getClass.getClassLoader.getResource(versionConf) + if (versionConfFileURL != null) { + versionConfPath = versionConfFileURL.getPath + } + } + + if (new File(versionConfPath).exists) { + logger.info( + s"*********************** Notice: The Linkis version file is $versionConf ! ******************" + ) + initConfig(config, versionConfPath) + configList.append(versionConfPath) + } else { + logger.warn( + s"**************** Notice: The Linkis version file $versionConf does not exist! *******************" + ) + } // init hot-load config task val hotLoadTask = new Runnable { override def run(): Unit = { diff --git a/linkis-commons/linkis-module/src/main/scala/org/apache/linkis/server/utils/LinkisMainHelper.scala b/linkis-commons/linkis-module/src/main/scala/org/apache/linkis/server/utils/LinkisMainHelper.scala index 7614bbef2a..d7d28361fe 100644 --- a/linkis-commons/linkis-module/src/main/scala/org/apache/linkis/server/utils/LinkisMainHelper.scala +++ b/linkis-commons/linkis-module/src/main/scala/org/apache/linkis/server/utils/LinkisMainHelper.scala @@ -61,7 +61,9 @@ object LinkisMainHelper { } resArr = resArr :+ s"--prometheus.endpoint=$prometheusEndpoint" } - return resArr + val version = CommonVars("version", "").getValue + resArr = resArr :+ s"--eureka.instance.metadata-map.linkis.app.version=$version" + resArr } } diff --git a/linkis-dist/package/conf/version.properties b/linkis-dist/package/conf/version.properties new file mode 100644 index 0000000000..05803eb75d --- /dev/null +++ b/linkis-dist/package/conf/version.properties @@ -0,0 +1,5 @@ +# batter to update this info when build linkis +#eureka metadata-map.linkis.app.version +version=Linkis-x.x.x-202306081009 +#build time information +build_time=2022-09-21 15:29.39 \ No newline at end of file From 8d3cb47b136a791b263266167015eadb41e9b0e5 Mon Sep 17 00:00:00 2001 From: Casion Date: Sun, 18 Jun 2023 22:32:46 +0800 Subject: [PATCH 165/261] Service support merge ps-data-source-manager ps-metadataquery into ps-datasource (#185) * Service Consolidation * update linkispsdatasource.list * code format spotless:apply --------- Co-authored-by: ahaoyao <731753729@qq.com> --- .../linkis/common/conf/BDPConfiguration.scala | 4 +--- .../linkis/rpc/conf/RPCConfiguration.scala | 10 ++++++++++ .../db/upgrade/1.4.0_schema/mysql/linkis_dml.sql | 16 ++++++++++++++++ .../linkis/gateway/parser/GatewayParser.scala | 9 +++++++++ 4 files changed, 36 insertions(+), 3 deletions(-) diff --git a/linkis-commons/linkis-common/src/main/scala/org/apache/linkis/common/conf/BDPConfiguration.scala b/linkis-commons/linkis-common/src/main/scala/org/apache/linkis/common/conf/BDPConfiguration.scala index b97b022a00..214beb881b 100644 --- a/linkis-commons/linkis-common/src/main/scala/org/apache/linkis/common/conf/BDPConfiguration.scala +++ b/linkis-commons/linkis-common/src/main/scala/org/apache/linkis/common/conf/BDPConfiguration.scala @@ -103,10 +103,9 @@ private[conf] object BDPConfiguration extends Logging { // load version conf val versionConf = sysProps.getOrElse("linkis.version.conf", DEFAULT_VERSION_FILE_NAME) - // version conf file path(env LINKIS_VERSION_CONF_FILE_PATH > classpath) var versionConfPath = sysProps.getOrElse("LINKIS_VERSION_CONF_FILE_PATH", "") - if(StringUtils.isBlank(versionConfPath)) { + if (StringUtils.isBlank(versionConfPath)) { logger.info( s"LINKIS_VERSION_CONF_FILE_PATH is empty, try to use version.properties file path from classpath" ) @@ -116,7 +115,6 @@ private[conf] object BDPConfiguration extends Logging { } } - if (new File(versionConfPath).exists) { logger.info( s"*********************** Notice: The Linkis version file is $versionConf ! ******************" diff --git a/linkis-commons/linkis-rpc/src/main/scala/org/apache/linkis/rpc/conf/RPCConfiguration.scala b/linkis-commons/linkis-rpc/src/main/scala/org/apache/linkis/rpc/conf/RPCConfiguration.scala index 32f12da273..d2dc6b1cc3 100644 --- a/linkis-commons/linkis-rpc/src/main/scala/org/apache/linkis/rpc/conf/RPCConfiguration.scala +++ b/linkis-commons/linkis-rpc/src/main/scala/org/apache/linkis/rpc/conf/RPCConfiguration.scala @@ -80,6 +80,16 @@ object RPCConfiguration { CommonVars("wds.linkis.gateway.conf.linkismanager.list", "linkisManager,engineplugin").getValue .split(",") + val LINKIS_DATASOURCE_SERVICE_NAME: CommonVars[String] = + CommonVars("linkis.gateway.conf.linkisdatasource.name", "linkis-ps-datasource") + + val LINKIS_DATASOURCE_SERVICE_LIST: Array[String] = + CommonVars( + "linkis.gateway.conf.linkisdatasource.list", + "data-source-manager,metadataquery,datasource" + ).getValue + .split(",") + val BDP_RPC_INSTANCE_ALIAS_SERVICE_REFRESH_INTERVAL: CommonVars[TimeType] = CommonVars("wds.linkis.rpc.instancealias.refresh.interval", new TimeType("3s")) diff --git a/linkis-dist/package/db/upgrade/1.4.0_schema/mysql/linkis_dml.sql b/linkis-dist/package/db/upgrade/1.4.0_schema/mysql/linkis_dml.sql index 07e811210b..a226d806db 100644 --- a/linkis-dist/package/db/upgrade/1.4.0_schema/mysql/linkis_dml.sql +++ b/linkis-dist/package/db/upgrade/1.4.0_schema/mysql/linkis_dml.sql @@ -1,3 +1,19 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ update linkis_ps_error_code set error_code = "01104" where error_regex ='ECM Insufficient number of instances'; update linkis_ps_error_code set error_code = "01105" where error_regex ='Cannot allocate memory'; diff --git a/linkis-spring-cloud-services/linkis-service-gateway/linkis-gateway-core/src/main/scala/org/apache/linkis/gateway/parser/GatewayParser.scala b/linkis-spring-cloud-services/linkis-service-gateway/linkis-gateway-core/src/main/scala/org/apache/linkis/gateway/parser/GatewayParser.scala index dd336809ab..bf56e10498 100644 --- a/linkis-spring-cloud-services/linkis-service-gateway/linkis-gateway-core/src/main/scala/org/apache/linkis/gateway/parser/GatewayParser.scala +++ b/linkis-spring-cloud-services/linkis-service-gateway/linkis-gateway-core/src/main/scala/org/apache/linkis/gateway/parser/GatewayParser.scala @@ -108,6 +108,12 @@ class DefaultGatewayParser(gatewayParsers: Array[GatewayParser]) extends Abstrac gatewayContext.getGatewayRoute.setRequestURI(path) } gatewayParsers.foreach(_.parse(gatewayContext)) + + /** + * Gateway forwarding logic: PublicService Service exists and is effective And then judge + * metadataquery Service, Continue to judge linkismanager Service, Final judgment + * linkispsdatasource Service + */ if (gatewayContext.getGatewayRoute.getServiceInstance == null) path match { case CLIENT_HEARTBEAT_REGEX(version) => if (sendResponseWhenNotMatchVersion(gatewayContext, version)) return @@ -126,6 +132,9 @@ class DefaultGatewayParser(gatewayParsers: Array[GatewayParser]) extends Abstrac RPCConfiguration.METADATAQUERY_SERVICE_APPLICATION_NAME.getValue } else if (RPCConfiguration.LINKIS_MANAGER_SERVICE_LIST.contains(serviceId)) { RPCConfiguration.LINKIS_MANAGER_SERVICE_NAME.getValue + // After the complete merge is completed, it needs to be removed + } else if (RPCConfiguration.LINKIS_DATASOURCE_SERVICE_LIST.contains(serviceId)) { + RPCConfiguration.LINKIS_DATASOURCE_SERVICE_NAME.getValue } else { serviceId } From b584d0db632bf8c68d317a7d65710829e0b598de Mon Sep 17 00:00:00 2001 From: peacewong Date: Sun, 18 Jun 2023 23:04:30 +0800 Subject: [PATCH 166/261] remove node scorer service invoke --- .../label/service/impl/DefaultNodeLabelService.scala | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/label/service/impl/DefaultNodeLabelService.scala b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/label/service/impl/DefaultNodeLabelService.scala index 399f19231f..a8dbe44352 100644 --- a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/label/service/impl/DefaultNodeLabelService.scala +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/label/service/impl/DefaultNodeLabelService.scala @@ -326,7 +326,10 @@ class DefaultNodeLabelService extends NodeLabelService with Logging { } if (null == necessaryLabels || necessaryLabels.isEmpty) { outNodeDegree.asScala.foreach { case (node, iLabels) => - matchInstanceAndLabels.put(new LabelScoreServiceInstance(node), iLabels.asInstanceOf) + matchInstanceAndLabels.put( + new LabelScoreServiceInstance(node), + iLabels.asInstanceOf[util.List[Label[_]]] + ) } } else { outNodeDegree.asScala.foreach { case (node, iLabels) => @@ -340,7 +343,10 @@ class DefaultNodeLabelService extends NodeLabelService with Logging { coreLabelKeys.asJava ) && coreLabelKeys.size == necessaryLabelKeys.size ) { - matchInstanceAndLabels.put(new LabelScoreServiceInstance(node), iLabels.asInstanceOf) + matchInstanceAndLabels.put( + new LabelScoreServiceInstance(node), + iLabels.asInstanceOf[util.List[Label[_]]] + ) } } } From 35bdb39b1b888454504d3b0efe787443ed8f0b32 Mon Sep 17 00:00:00 2001 From: peacewong Date: Mon, 19 Jun 2023 14:50:24 +0800 Subject: [PATCH 167/261] FIX conflict --- .../impl/DefaultNodeManagerPersistence.java | 31 ++++++++++--------- .../mapper/common/NodeManagerMapper.xml | 13 +++++--- 2 files changed, 26 insertions(+), 18 deletions(-) diff --git a/linkis-computation-governance/linkis-manager/linkis-manager-persistence/src/main/java/org/apache/linkis/manager/persistence/impl/DefaultNodeManagerPersistence.java b/linkis-computation-governance/linkis-manager/linkis-manager-persistence/src/main/java/org/apache/linkis/manager/persistence/impl/DefaultNodeManagerPersistence.java index cb4ae406a9..061588ac65 100644 --- a/linkis-computation-governance/linkis-manager/linkis-manager-persistence/src/main/java/org/apache/linkis/manager/persistence/impl/DefaultNodeManagerPersistence.java +++ b/linkis-computation-governance/linkis-manager/linkis-manager-persistence/src/main/java/org/apache/linkis/manager/persistence/impl/DefaultNodeManagerPersistence.java @@ -364,21 +364,24 @@ public List getEngineNodeByServiceInstance(List ser @Override public List getNodesByOwnerList(List ownerlist) { - List nodeInstances = nodeManagerMapper.getNodeInstancesByOwnerList(ownerlist); List persistenceNodeEntitys = new ArrayList<>(); - if (!nodeInstances.isEmpty()) { - for (PersistenceNode persistenceNode : nodeInstances) { - PersistenceNodeEntity persistenceNodeEntity = new PersistenceNodeEntity(); - ServiceInstance serviceInstance = new ServiceInstance(); - serviceInstance.setApplicationName(persistenceNode.getName()); - serviceInstance.setInstance(persistenceNode.getInstance()); - persistenceNodeEntity.setServiceInstance(serviceInstance); - persistenceNodeEntity.setMark(persistenceNode.getMark()); - persistenceNodeEntity.setIdentifier(persistenceNode.getIdentifier()); - persistenceNodeEntity.setTicketId(persistenceNode.getTicketId()); - persistenceNodeEntity.setOwner(persistenceNode.getOwner()); - persistenceNodeEntity.setStartTime(persistenceNode.getCreateTime()); - persistenceNodeEntitys.add(persistenceNodeEntity); + if (CollectionUtils.isNotEmpty(ownerlist)) { + List nodeInstances = + nodeManagerMapper.getNodeInstancesByOwnerList(ownerlist); + if (CollectionUtils.isNotEmpty(nodeInstances)) { + for (PersistenceNode persistenceNode : nodeInstances) { + PersistenceNodeEntity persistenceNodeEntity = new PersistenceNodeEntity(); + ServiceInstance serviceInstance = new ServiceInstance(); + serviceInstance.setApplicationName(persistenceNode.getName()); + serviceInstance.setInstance(persistenceNode.getInstance()); + persistenceNodeEntity.setServiceInstance(serviceInstance); + persistenceNodeEntity.setMark(persistenceNode.getMark()); + persistenceNodeEntity.setOwner(persistenceNode.getOwner()); + persistenceNodeEntity.setStartTime(persistenceNode.getCreateTime()); + persistenceNodeEntity.setIdentifier(persistenceNode.getIdentifier()); + persistenceNodeEntity.setTicketId(persistenceNode.getTicketId()); + persistenceNodeEntitys.add(persistenceNodeEntity); + } } } return persistenceNodeEntitys; diff --git a/linkis-computation-governance/linkis-manager/linkis-manager-persistence/src/main/resources/mapper/common/NodeManagerMapper.xml b/linkis-computation-governance/linkis-manager/linkis-manager-persistence/src/main/resources/mapper/common/NodeManagerMapper.xml index 3dca427c81..2a02abeae1 100644 --- a/linkis-computation-governance/linkis-manager/linkis-manager-persistence/src/main/resources/mapper/common/NodeManagerMapper.xml +++ b/linkis-computation-governance/linkis-manager/linkis-manager-persistence/src/main/resources/mapper/common/NodeManagerMapper.xml @@ -181,10 +181,15 @@ From af30f6c98646d150e410fcd39f094502504fb089 Mon Sep 17 00:00:00 2001 From: peacewong Date: Mon, 19 Jun 2023 16:13:16 +0800 Subject: [PATCH 168/261] Fix build issue --- .../org/apache/linkis/storage/utils/FileSystemUtils.scala | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/utils/FileSystemUtils.scala b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/utils/FileSystemUtils.scala index 68296401d3..0e3949b2b3 100644 --- a/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/utils/FileSystemUtils.scala +++ b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/utils/FileSystemUtils.scala @@ -22,10 +22,9 @@ import org.apache.linkis.common.utils.{Logging, Utils} import org.apache.linkis.storage.FSFactory import org.apache.linkis.storage.fs.FileSystem import org.apache.linkis.storage.fs.impl.LocalFileSystem -import org.apache.linkis.storage.source.{FileSource, FileSource$} +import org.apache.linkis.storage.source.FileSource import org.apache.commons.io.IOUtils -import org.apache.commons.math3.util.Pair import java.io.IOException import java.util From 0d93f08d387fe4acdc97c471a0ca9d662aa7aa22 Mon Sep 17 00:00:00 2001 From: casionone Date: Mon, 19 Jun 2023 16:47:03 +0800 Subject: [PATCH 169/261] add mapper --- .../dao/ConfigKeyLimitForUserMapper.java | 100 +++++ .../configuration/dao/ConfigMapper.java | 13 +- .../dao/TemplateConfigKeyMapper.java | 103 +++++ .../configuration/entity/ConfigKey.java | 11 + .../entity/ConfigKeyLimitForUser.java | 307 ++++++++++++++ .../entity/TemplateConfigKey.java | 391 ++++++++++++++++++ .../entity/TemplateConfigKeyVo.java | 61 +++ .../restful/api/ConfigurationRestfulApi.java | 9 + .../api/ConfigurationTemplateRestfulApi.java | 140 +++++++ .../service/ConfigKeyService.java | 5 + .../service/TemplateConfigKeyService.java | 31 ++ .../service/impl/ConfigKeyServiceImpl.java | 5 + .../impl/TemplateConfigKeyServiceImpl.java | 139 +++++++ .../common/ConfigKeyLimitForUserMapper.xml | 173 ++++++++ .../resources/mapper/common/ConfigMapper.xml | 38 +- .../mapper/common/TemplateConfigKeyMapper.xml | 228 ++++++++++ .../service/ConfigurationService.scala | 5 - .../configuration/dao/ConfigMapperTest.java | 6 - 18 files changed, 1742 insertions(+), 23 deletions(-) create mode 100644 linkis-public-enhancements/linkis-configuration/src/main/java/org/apache/linkis/configuration/dao/ConfigKeyLimitForUserMapper.java create mode 100644 linkis-public-enhancements/linkis-configuration/src/main/java/org/apache/linkis/configuration/dao/TemplateConfigKeyMapper.java create mode 100644 linkis-public-enhancements/linkis-configuration/src/main/java/org/apache/linkis/configuration/entity/ConfigKeyLimitForUser.java create mode 100644 linkis-public-enhancements/linkis-configuration/src/main/java/org/apache/linkis/configuration/entity/TemplateConfigKey.java create mode 100644 linkis-public-enhancements/linkis-configuration/src/main/java/org/apache/linkis/configuration/entity/TemplateConfigKeyVo.java create mode 100644 linkis-public-enhancements/linkis-configuration/src/main/java/org/apache/linkis/configuration/restful/api/ConfigurationTemplateRestfulApi.java create mode 100644 linkis-public-enhancements/linkis-configuration/src/main/java/org/apache/linkis/configuration/service/TemplateConfigKeyService.java create mode 100644 linkis-public-enhancements/linkis-configuration/src/main/java/org/apache/linkis/configuration/service/impl/TemplateConfigKeyServiceImpl.java create mode 100644 linkis-public-enhancements/linkis-configuration/src/main/resources/mapper/common/ConfigKeyLimitForUserMapper.xml create mode 100644 linkis-public-enhancements/linkis-configuration/src/main/resources/mapper/common/TemplateConfigKeyMapper.xml diff --git a/linkis-public-enhancements/linkis-configuration/src/main/java/org/apache/linkis/configuration/dao/ConfigKeyLimitForUserMapper.java b/linkis-public-enhancements/linkis-configuration/src/main/java/org/apache/linkis/configuration/dao/ConfigKeyLimitForUserMapper.java new file mode 100644 index 0000000000..2e7626a194 --- /dev/null +++ b/linkis-public-enhancements/linkis-configuration/src/main/java/org/apache/linkis/configuration/dao/ConfigKeyLimitForUserMapper.java @@ -0,0 +1,100 @@ +package org.apache.linkis.configuration.dao; + +import java.util.List; +import org.apache.ibatis.annotations.Param; +import org.apache.linkis.configuration.entity.ConfigKeyLimitForUser; + + +/** + * config_key_limit_for_user表的dao接口类 + * @Description + * @version 1.0 + * @author webank + */ +public interface ConfigKeyLimitForUserMapper { + + /** + * 根据主键删除数据库的记录 + * + * @param id + */ + int deleteByPrimaryKey(Long id); + + /** + * 新写入数据库记录 + * + * @param configKeyLimitForUser + */ + int insert(ConfigKeyLimitForUser configKeyLimitForUser); + + /** + * 批量插入记录，建议List一次不要超过1000条 + * + * @param list + */ + int batchInsertList(List list); + + /** + * 动态字段,写入数据库记录 + * + * @param configKeyLimitForUser + */ + int insertSelective(ConfigKeyLimitForUser configKeyLimitForUser); + + /** + * 根据指定主键获取一条数据库记录 + * + * @param id + */ + ConfigKeyLimitForUser selectByPrimaryKey(Long id); + + /** + * 查询分页数据条数 - 示例方法 + * + * @param id + */ + int selectCountByPage(Long id); + + /** + * 查询分页数据列表 - 示例方法 + * public DataPage selectByPage(Id id, int pageNo, int pageSize) { + * if (pageNo > 100) { + * pageNo = 100; + * } + * if (pageNo < 1) { + * pageNo = 1; + * } + * if (pageSize > 50) { + * pageSize = 50; + * } + * if (pageSize < 1) { + * pageSize = 1; + * } + * int totalCount = configKeyLimitForUserDAO.selectCountByPage(id); + * List list = configKeyLimitForUserDAO.selectListByPage(id, pageNo, pageSize); + * DataPage dp = new DataPage<>(list, pageSize, pageNo, totalCount); + * return dp; + * } + * + * @param id + * @param pageNo + * @param pageSize + */ + List selectListByPage(@Param("id") Long id, @Param("pageNo") int pageNo, @Param("pageSize") int pageSize); + + /** + * 动态字段,根据主键来更新符合条件的数据库记录 + * + * @param configKeyLimitForUser + */ + int updateByPrimaryKeySelective(ConfigKeyLimitForUser configKeyLimitForUser); + + /** + * 根据主键来更新符合条件的数据库记录 + * + * @param configKeyLimitForUser + */ + int updateByPrimaryKey(ConfigKeyLimitForUser configKeyLimitForUser); + + // === 下方为用户自定义模块,下次生成会保留 === +} diff --git a/linkis-public-enhancements/linkis-configuration/src/main/java/org/apache/linkis/configuration/dao/ConfigMapper.java b/linkis-public-enhancements/linkis-configuration/src/main/java/org/apache/linkis/configuration/dao/ConfigMapper.java index 6b6b15a65c..cb9c3e0f97 100644 --- a/linkis-public-enhancements/linkis-configuration/src/main/java/org/apache/linkis/configuration/dao/ConfigMapper.java +++ b/linkis-public-enhancements/linkis-configuration/src/main/java/org/apache/linkis/configuration/dao/ConfigMapper.java @@ -28,17 +28,11 @@ public interface ConfigMapper { - List getConfigByEngineUserCreator( - @Param("engineType") String engineType, - @Param("creator") String creator, - @Param("userName") String userName); List getConfigKeyByLabelIds(@Param("ids") List ids); List getConfigKeyValueByLabelId(@Param("labelId") Integer labelId); - Long selectAppIDByAppName(@Param("name") String appName); - void insertValue(ConfigValue configValue); ConfigValue getConfigValueById(@Param("id") Long id); @@ -57,9 +51,14 @@ List getConfigByEngineUserCreator( List selectKeyByKeyName(@Param("keyName") String keyName); + List selectKeyByEngineType(@Param("engineType") String engineType); + + List selectKeyByEngineTypeAndKeyList(@Param("engineType") String engineType, + @Param("KeyList") List keyList); + + List listKeyByStringValue(@Param("stringValue") String stringValue); - void insertCreator(String creator); List getCategory(); diff --git a/linkis-public-enhancements/linkis-configuration/src/main/java/org/apache/linkis/configuration/dao/TemplateConfigKeyMapper.java b/linkis-public-enhancements/linkis-configuration/src/main/java/org/apache/linkis/configuration/dao/TemplateConfigKeyMapper.java new file mode 100644 index 0000000000..1e5967871d --- /dev/null +++ b/linkis-public-enhancements/linkis-configuration/src/main/java/org/apache/linkis/configuration/dao/TemplateConfigKeyMapper.java @@ -0,0 +1,103 @@ +package org.apache.linkis.configuration.dao; + +import java.util.List; +import org.apache.ibatis.annotations.Param; +import org.apache.linkis.configuration.entity.TemplateConfigKey; + +/** + * template_config_key表的dao接口类 + * @Description + * @version 1.0 + * @author webank + */ +public interface TemplateConfigKeyMapper { + + /** + * 根据主键删除数据库的记录 + * + * @param id + */ + int deleteByPrimaryKey(Long id); + + /** + * 新写入数据库记录 + * + * @param templateConfigKey + */ + int insert(TemplateConfigKey templateConfigKey); + + /** + * 批量插入记录，建议List一次不要超过1000条 + * + * @param list + */ + int batchInsertList(List list); + + /** + * 动态字段,写入数据库记录 + * + * @param templateConfigKey + */ + int insertSelective(TemplateConfigKey templateConfigKey); + + /** + * 根据指定主键获取一条数据库记录 + * + * @param id + */ + TemplateConfigKey selectByPrimaryKey(Long id); + + /** + * 查询分页数据条数 - 示例方法 + * + * @param id + */ + int selectCountByPage(Long id); + + /** + * 查询分页数据列表 - 示例方法 + * public DataPage selectByPage(Id id, int pageNo, int pageSize) { + * if (pageNo > 100) { + * pageNo = 100; + * } + * if (pageNo < 1) { + * pageNo = 1; + * } + * if (pageSize > 50) { + * pageSize = 50; + * } + * if (pageSize < 1) { + * pageSize = 1; + * } + * int totalCount = templateConfigKeyDAO.selectCountByPage(id); + * List list = templateConfigKeyDAO.selectListByPage(id, pageNo, pageSize); + * DataPage dp = new DataPage<>(list, pageSize, pageNo, totalCount); + * return dp; + * } + * + * @param id + * @param pageNo + * @param pageSize + */ + List selectListByPage(@Param("id") Long id, @Param("pageNo") int pageNo, @Param("pageSize") int pageSize); + + /** + * 动态字段,根据主键来更新符合条件的数据库记录 + * + * @param templateConfigKey + */ + int updateByPrimaryKeySelective(TemplateConfigKey templateConfigKey); + + /** + * 根据主键来更新符合条件的数据库记录 + * + * @param templateConfigKey + */ + int updateByPrimaryKey(TemplateConfigKey templateConfigKey); + + // === 下方为用户自定义模块,下次生成会保留 === + List selectListByTemplateUuid(@Param("templateUuid") String templateUuid); + + + int deleteByTemplateUuidAndKeyIdList(@Param("templateUuid") String templateUuid, @Param("KeyIdList")List KeyIdList); +} diff --git a/linkis-public-enhancements/linkis-configuration/src/main/java/org/apache/linkis/configuration/entity/ConfigKey.java b/linkis-public-enhancements/linkis-configuration/src/main/java/org/apache/linkis/configuration/entity/ConfigKey.java index 1e26252a7c..8c54e4663c 100644 --- a/linkis-public-enhancements/linkis-configuration/src/main/java/org/apache/linkis/configuration/entity/ConfigKey.java +++ b/linkis-public-enhancements/linkis-configuration/src/main/java/org/apache/linkis/configuration/entity/ConfigKey.java @@ -43,6 +43,9 @@ public class ConfigKey { private String treeName; + // 0 none/ 1 with mix /2 with max / 3 min and max both + private Integer boundaryType; + public String getEngineType() { return engineType; } @@ -138,4 +141,12 @@ public Integer getLevel() { public void setLevel(Integer level) { this.level = level; } + + public Integer getBoundaryType() { + return boundaryType; + } + + public void setBoundaryType(Integer boundaryType) { + this.boundaryType = boundaryType; + } } diff --git a/linkis-public-enhancements/linkis-configuration/src/main/java/org/apache/linkis/configuration/entity/ConfigKeyLimitForUser.java b/linkis-public-enhancements/linkis-configuration/src/main/java/org/apache/linkis/configuration/entity/ConfigKeyLimitForUser.java new file mode 100644 index 0000000000..c3685e4902 --- /dev/null +++ b/linkis-public-enhancements/linkis-configuration/src/main/java/org/apache/linkis/configuration/entity/ConfigKeyLimitForUser.java @@ -0,0 +1,307 @@ +package org.apache.linkis.configuration.entity; + +import java.util.Date; + +/** + * config_key_limit_for_user表的entity类 + * @Description + * @version 1.0 + * @author webank + * @see http://10.107.99.84:53661/wego-rad/site/plugin/weup-tool/mybatisGen.html + * @see http://km.weoa.com/group/wesmart/article/10467 + */ +public class ConfigKeyLimitForUser { + + /** + * 表字段 : id + * 字段类型 : bigint(19) + */ + private Long id; + + /** + * 用户名 + * 表字段 : user_name + * 字段类型 : varchar(50) + */ + private String userName; + + /** + * 组合标签 combined_userCreator_engineType 如 hadoop-IDE,spark-2.4.3 + * 表字段 : combined_label_value + * 字段类型 : varchar(200) + */ + private String combinedLabelValue; + + /** + * id of linkis_ps_configuration_config_key + * 表字段 : key_id + * 字段类型 : bigint(19) + */ + private Long keyId; + + /** + * uuid 第三方侧记录的模板id + * 表字段 : latest_update_template_uuid + * 字段类型 : varchar(34) + */ + private String latestUpdateTemplateUuid; + + /** + * 是否有效 预留 Y/N + * 表字段 : is_valid + * 字段类型 : varchar(2) + */ + private String isValid; + + /** + * 创建人 + * 表字段 : create_by + * 字段类型 : varchar(50) + */ + private String createBy; + + /** + * create time + * 表字段 : create_time + * 字段类型 : timestamp(19) + * 默认值 : CURRENT_TIMESTAMP + */ + private Date createTime; + + /** + * 更新人 + * 表字段 : update_by + * 字段类型 : varchar(50) + */ + private String updateBy; + + /** + * update time + * 表字段 : update_time + * 字段类型 : timestamp(19) + * 默认值 : CURRENT_TIMESTAMP + */ + private Date updateTime; + + /** + * 获取：
+ * 字段：id + * + * @return + */ + public Long getId() { + return id; + } + + /** + * 设置：
+ * 字段：id + * + * @param + */ + public void setId(Long id) { + this.id = id; + } + + /** + * 获取：用户名
+ * 字段：user_name + * + * @return + */ + public String getUserName() { + return userName; + } + + /** + * 设置：用户名
+ * 字段：user_name + * + * @param + */ + public void setUserName(String userName) { + this.userName = userName; + } + + /** + * 获取：组合标签 combined_userCreator_engineType 如 hadoop-IDE,spark-2.4.3
+ * 字段：combined_label_value + * + * @return + */ + public String getCombinedLabelValue() { + return combinedLabelValue; + } + + /** + * 设置：组合标签 combined_userCreator_engineType 如 hadoop-IDE,spark-2.4.3
+ * 字段：combined_label_value + * + * @param + */ + public void setCombinedLabelValue(String combinedLabelValue) { + this.combinedLabelValue = combinedLabelValue; + } + + /** + * 获取：id of linkis_ps_configuration_config_key
+ * 字段：key_id + * + * @return + */ + public Long getKeyId() { + return keyId; + } + + /** + * 设置：id of linkis_ps_configuration_config_key
+ * 字段：key_id + * + * @param + */ + public void setKeyId(Long keyId) { + this.keyId = keyId; + } + + /** + * 获取：uuid 第三方侧记录的模板id
+ * 字段：latest_update_template_uuid + * + * @return + */ + public String getLatestUpdateTemplateUuid() { + return latestUpdateTemplateUuid; + } + + /** + * 设置：uuid 第三方侧记录的模板id
+ * 字段：latest_update_template_uuid + * + * @param + */ + public void setLatestUpdateTemplateUuid(String latestUpdateTemplateUuid) { + this.latestUpdateTemplateUuid = latestUpdateTemplateUuid; + } + + /** + * 获取：是否有效 预留 Y/N
+ * 字段：is_valid + * + * @return + */ + public String getIsValid() { + return isValid; + } + + /** + * 设置：是否有效 预留 Y/N
+ * 字段：is_valid + * + * @param + */ + public void setIsValid(String isValid) { + this.isValid = isValid; + } + + /** + * 获取：创建人
+ * 字段：create_by + * + * @return + */ + public String getCreateBy() { + return createBy; + } + + /** + * 设置：创建人
+ * 字段：create_by + * + * @param + */ + public void setCreateBy(String createBy) { + this.createBy = createBy; + } + + /** + * 获取：create time
+ * 字段：create_time + * + * @return + */ + public Date getCreateTime() { + return createTime; + } + + /** + * 设置：create time
+ * 字段：create_time + * + * @param + */ + public void setCreateTime(Date createTime) { + this.createTime = createTime; + } + + /** + * 获取：更新人
+ * 字段：update_by + * + * @return + */ + public String getUpdateBy() { + return updateBy; + } + + /** + * 设置：更新人
+ * 字段：update_by + * + * @param + */ + public void setUpdateBy(String updateBy) { + this.updateBy = updateBy; + } + + /** + * 获取：update time
+ * 字段：update_time + * + * @return + */ + public Date getUpdateTime() { + return updateTime; + } + + /** + * 设置：update time
+ * 字段：update_time + * + * @param + */ + public void setUpdateTime(Date updateTime) { + this.updateTime = updateTime; + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + sb.append(getClass().getSimpleName()); + sb.append(" ["); + sb.append("Hash = ").append(hashCode()); + sb.append(", id=").append(id); + sb.append(", userName=").append(userName); + sb.append(", combinedLabelValue=").append(combinedLabelValue); + sb.append(", keyId=").append(keyId); + sb.append(", latestUpdateTemplateUuid=").append(latestUpdateTemplateUuid); + sb.append(", isValid=").append(isValid); + sb.append(", createBy=").append(createBy); + sb.append(", createTime=").append(createTime); + sb.append(", updateBy=").append(updateBy); + sb.append(", updateTime=").append(updateTime); + sb.append(']'); + return sb.toString(); + } + + // === 下方为用户自定义模块,下次生成会保留 === +} \ No newline at end of file diff --git a/linkis-public-enhancements/linkis-configuration/src/main/java/org/apache/linkis/configuration/entity/TemplateConfigKey.java b/linkis-public-enhancements/linkis-configuration/src/main/java/org/apache/linkis/configuration/entity/TemplateConfigKey.java new file mode 100644 index 0000000000..3df6532037 --- /dev/null +++ b/linkis-public-enhancements/linkis-configuration/src/main/java/org/apache/linkis/configuration/entity/TemplateConfigKey.java @@ -0,0 +1,391 @@ +package org.apache.linkis.configuration.entity; + +import java.util.Date; + +/** + * template_config_key表的entity类 + * @Description + * @version 1.0 + * @author webank + * @see http://10.107.99.84:53661/wego-rad/site/plugin/weup-tool/mybatisGen.html + * @see http://km.weoa.com/group/wesmart/article/10467 + */ +public class TemplateConfigKey { + + /** + * 表字段 : id + * 字段类型 : bigint(19) + */ + private Long id; + + /** + * 配置模板名称 冗余存储 + * 表字段 : template_name + * 字段类型 : varchar(200) + */ + private String templateName; + + /** + * uuid 第三方侧记录的模板id + * 表字段 : template_uuid + * 字段类型 : varchar(34) + */ + private String templateUuid; + + /** + * id of linkis_ps_configuration_config_key + * 表字段 : key_id + * 字段类型 : bigint(19) + */ + private Long keyId; + + /** + * 配置值 + * 表字段 : config_value + * 字段类型 : varchar(200) + */ + private String configValue; + + /** + * 上限值 + * 表字段 : max_value + * 字段类型 : varchar(50) + */ + private String maxValue; + + /** + * 下限值（预留） + * 表字段 : min_value + * 字段类型 : varchar(50) + */ + private String minValue; + + /** + * 校验正则(预留) + * 表字段 : validate_range + * 字段类型 : varchar(50) + */ + private String validateRange; + + /** + * 是否有效 预留 Y/N + * 表字段 : is_valid + * 字段类型 : varchar(2) + */ + private String isValid; + + /** + * 创建人 + * 表字段 : create_by + * 字段类型 : varchar(50) + */ + private String createBy; + + /** + * create time + * 表字段 : create_time + * 字段类型 : timestamp(19) + * 默认值 : CURRENT_TIMESTAMP + */ + private Date createTime; + + /** + * 更新人 + * 表字段 : update_by + * 字段类型 : varchar(50) + */ + private String updateBy; + + /** + * update time + * 表字段 : update_time + * 字段类型 : timestamp(19) + * 默认值 : CURRENT_TIMESTAMP + */ + private Date updateTime; + + /** + * 获取：
+ * 字段：id + * + * @return + */ + public Long getId() { + return id; + } + + /** + * 设置：
+ * 字段：id + * + * @param + */ + public void setId(Long id) { + this.id = id; + } + + /** + * 获取：配置模板名称 冗余存储
+ * 字段：template_name + * + * @return + */ + public String getTemplateName() { + return templateName; + } + + /** + * 设置：配置模板名称 冗余存储
+ * 字段：template_name + * + * @param + */ + public void setTemplateName(String templateName) { + this.templateName = templateName; + } + + /** + * 获取：uuid 第三方侧记录的模板id
+ * 字段：template_uuid + * + * @return + */ + public String getTemplateUuid() { + return templateUuid; + } + + /** + * 设置：uuid 第三方侧记录的模板id
+ * 字段：template_uuid + * + * @param + */ + public void setTemplateUuid(String templateUuid) { + this.templateUuid = templateUuid; + } + + /** + * 获取：id of linkis_ps_configuration_config_key
+ * 字段：key_id + * + * @return + */ + public Long getKeyId() { + return keyId; + } + + /** + * 设置：id of linkis_ps_configuration_config_key
+ * 字段：key_id + * + * @param + */ + public void setKeyId(Long keyId) { + this.keyId = keyId; + } + + /** + * 获取：配置值
+ * 字段：config_value + * + * @return + */ + public String getConfigValue() { + return configValue; + } + + /** + * 设置：配置值
+ * 字段：config_value + * + * @param + */ + public void setConfigValue(String configValue) { + this.configValue = configValue; + } + + /** + * 获取：上限值
+ * 字段：max_value + * + * @return + */ + public String getMaxValue() { + return maxValue; + } + + /** + * 设置：上限值
+ * 字段：max_value + * + * @param + */ + public void setMaxValue(String maxValue) { + this.maxValue = maxValue; + } + + /** + * 获取：下限值（预留）
+ * 字段：min_value + * + * @return + */ + public String getMinValue() { + return minValue; + } + + /** + * 设置：下限值（预留）
+ * 字段：min_value + * + * @param + */ + public void setMinValue(String minValue) { + this.minValue = minValue; + } + + /** + * 获取：校验正则(预留)
+ * 字段：validate_range + * + * @return + */ + public String getValidateRange() { + return validateRange; + } + + /** + * 设置：校验正则(预留)
+ * 字段：validate_range + * + * @param + */ + public void setValidateRange(String validateRange) { + this.validateRange = validateRange; + } + + /** + * 获取：是否有效 预留 Y/N
+ * 字段：is_valid + * + * @return + */ + public String getIsValid() { + return isValid; + } + + /** + * 设置：是否有效 预留 Y/N
+ * 字段：is_valid + * + * @param + */ + public void setIsValid(String isValid) { + this.isValid = isValid; + } + + /** + * 获取：创建人
+ * 字段：create_by + * + * @return + */ + public String getCreateBy() { + return createBy; + } + + /** + * 设置：创建人
+ * 字段：create_by + * + * @param + */ + public void setCreateBy(String createBy) { + this.createBy = createBy; + } + + /** + * 获取：create time
+ * 字段：create_time + * + * @return + */ + public Date getCreateTime() { + return createTime; + } + + /** + * 设置：create time
+ * 字段：create_time + * + * @param + */ + public void setCreateTime(Date createTime) { + this.createTime = createTime; + } + + /** + * 获取：更新人
+ * 字段：update_by + * + * @return + */ + public String getUpdateBy() { + return updateBy; + } + + /** + * 设置：更新人
+ * 字段：update_by + * + * @param + */ + public void setUpdateBy(String updateBy) { + this.updateBy = updateBy; + } + + /** + * 获取：update time
+ * 字段：update_time + * + * @return + */ + public Date getUpdateTime() { + return updateTime; + } + + /** + * 设置：update time
+ * 字段：update_time + * + * @param + */ + public void setUpdateTime(Date updateTime) { + this.updateTime = updateTime; + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + sb.append(getClass().getSimpleName()); + sb.append(" ["); + sb.append("Hash = ").append(hashCode()); + sb.append(", id=").append(id); + sb.append(", templateName=").append(templateName); + sb.append(", templateUuid=").append(templateUuid); + sb.append(", keyId=").append(keyId); + sb.append(", configValue=").append(configValue); + sb.append(", maxValue=").append(maxValue); + sb.append(", minValue=").append(minValue); + sb.append(", validateRange=").append(validateRange); + sb.append(", isValid=").append(isValid); + sb.append(", createBy=").append(createBy); + sb.append(", createTime=").append(createTime); + sb.append(", updateBy=").append(updateBy); + sb.append(", updateTime=").append(updateTime); + sb.append(']'); + return sb.toString(); + } + + // === 下方为用户自定义模块,下次生成会保留 === +} \ No newline at end of file diff --git a/linkis-public-enhancements/linkis-configuration/src/main/java/org/apache/linkis/configuration/entity/TemplateConfigKeyVo.java b/linkis-public-enhancements/linkis-configuration/src/main/java/org/apache/linkis/configuration/entity/TemplateConfigKeyVo.java new file mode 100644 index 0000000000..911a2b0b38 --- /dev/null +++ b/linkis-public-enhancements/linkis-configuration/src/main/java/org/apache/linkis/configuration/entity/TemplateConfigKeyVo.java @@ -0,0 +1,61 @@ +package org.apache.linkis.configuration.entity; + + + +public class TemplateConfigKeyVo { + + /** + * 表字段 : id + * 字段类型 : bigint(19) + */ + private Long id; + + private String key; + + /** + * 配置值 + * 表字段 : config_value + * 字段类型 : varchar(200) + */ + private String configValue; + + /** + * 上限值 + * 表字段 : max_value + * 字段类型 : varchar(50) + */ + private String maxValue; + + public Long getId() { + return id; + } + + public void setId(Long id) { + this.id = id; + } + + + public String getConfigValue() { + return configValue; + } + + public void setConfigValue(String configValue) { + this.configValue = configValue; + } + + public String getMaxValue() { + return maxValue; + } + + public void setMaxValue(String maxValue) { + this.maxValue = maxValue; + } + + public String getKey() { + return key; + } + + public void setKey(String key) { + this.key = key; + } +} \ No newline at end of file diff --git a/linkis-public-enhancements/linkis-configuration/src/main/java/org/apache/linkis/configuration/restful/api/ConfigurationRestfulApi.java b/linkis-public-enhancements/linkis-configuration/src/main/java/org/apache/linkis/configuration/restful/api/ConfigurationRestfulApi.java index 283960d5df..eb1e314307 100644 --- a/linkis-public-enhancements/linkis-configuration/src/main/java/org/apache/linkis/configuration/restful/api/ConfigurationRestfulApi.java +++ b/linkis-public-enhancements/linkis-configuration/src/main/java/org/apache/linkis/configuration/restful/api/ConfigurationRestfulApi.java @@ -155,6 +155,15 @@ public Message getCategory(HttpServletRequest req) { return Message.ok().data("Category", categoryLabelList); } + @ApiOperation(value = "getItemList", notes = "get configuration list by engineType", response = Message.class) + @RequestMapping(path = "/getItemList", method = RequestMethod.GET) + public Message getItemList(HttpServletRequest req, @RequestParam(value = "engineType") String engineType) throws ConfigurationException { + String userName = ModuleUserUtils.getOperationUser(req, "getItemList with engineType:" + engineType); + List result= configKeyService.getConfigKeyList(engineType); + return Message.ok().data("list", result); + } + + @ApiOperation( value = "createFirstCategory", notes = "create first category", diff --git a/linkis-public-enhancements/linkis-configuration/src/main/java/org/apache/linkis/configuration/restful/api/ConfigurationTemplateRestfulApi.java b/linkis-public-enhancements/linkis-configuration/src/main/java/org/apache/linkis/configuration/restful/api/ConfigurationTemplateRestfulApi.java new file mode 100644 index 0000000000..5116609ee1 --- /dev/null +++ b/linkis-public-enhancements/linkis-configuration/src/main/java/org/apache/linkis/configuration/restful/api/ConfigurationTemplateRestfulApi.java @@ -0,0 +1,140 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.configuration.restful.api; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.github.xiaoymin.knife4j.annotations.ApiOperationSupport; +import io.swagger.annotations.Api; +import io.swagger.annotations.ApiImplicitParam; +import io.swagger.annotations.ApiImplicitParams; +import io.swagger.annotations.ApiOperation; +import org.apache.commons.lang3.StringUtils; +import org.apache.linkis.common.conf.Configuration; +import org.apache.linkis.common.utils.JsonUtils; +import org.apache.linkis.configuration.entity.*; +import org.apache.linkis.configuration.exception.ConfigurationException; +import org.apache.linkis.configuration.service.CategoryService; +import org.apache.linkis.configuration.service.ConfigKeyService; +import org.apache.linkis.configuration.service.ConfigurationService; +import org.apache.linkis.configuration.util.ConfigurationConfiguration; +import org.apache.linkis.configuration.util.JsonNodeUtil; +import org.apache.linkis.configuration.util.LabelEntityParser; +import org.apache.linkis.manager.label.entity.engine.EngineTypeLabel; +import org.apache.linkis.manager.label.entity.engine.UserCreatorLabel; +import org.apache.linkis.manager.label.utils.LabelUtils; +import org.apache.linkis.server.BDPJettyServerHelper; +import org.apache.linkis.server.Message; +import org.apache.linkis.server.utils.ModuleUserUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.web.bind.annotation.*; + +import javax.servlet.http.HttpServletRequest; +import java.io.IOException; +import java.text.MessageFormat; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static org.apache.linkis.configuration.errorcode.LinkisConfigurationErrorCodeSummary.*; + +@Api(tags = "configuration template") +@RestController +@RequestMapping(path = "/configuration/template") +public class ConfigurationTemplateRestfulApi { + + private static final Logger logger = LoggerFactory.getLogger(ConfigurationTemplateRestfulApi.class); + + @Autowired private ConfigurationService configurationService; + + @Autowired private CategoryService categoryService; + + @Autowired private ConfigKeyService configKeyService; + + @ApiOperation(value = "updateKeyMapping", notes = "query engineconn info list", response = Message.class) + @ApiImplicitParams({ + @ApiImplicitParam(name = "templateUid", dataType = "String", required = true, value = "templateUid"), + @ApiImplicitParam(name = "templateName", dataType = "String", required = true, value = "engine type"), + @ApiImplicitParam(name = "engineType", dataType = "String", required = true,value = "String"), + @ApiImplicitParam(name = "operator", dataType = "String", value = "operator"), + @ApiImplicitParam(name = "isFullMode", dataType = "Boolbean", value = "isFullMode"), + @ApiImplicitParam(name = "itemList", dataType = "Array", value = "itemList"), + }) + @RequestMapping(path = "/updateKeyMapping", method = RequestMethod.POST) + public Message updateKeyMapping(HttpServletRequest req, @RequestBody JsonNode jsonNode) { + String username = ModuleUserUtils.getOperationUser(req, "updateKeyMapping"); + String token = ModuleUserUtils.getToken(req); + // check special admin token + if (StringUtils.isNotBlank(token)) { + if (!Configuration.isAdminToken(token)) { + logger.warn("Token:{} has no permission to updateKeyMapping.", token); + return Message.error("Token:" + token + " has no permission to updateKeyMapping."); + } + } else if (!Configuration.isAdmin(username)) { + logger.warn("User:{} has no permission to updateKeyMapping.", username); + return Message.error("User:" + username + " has no permission to updateKeyMapping."); + } + + String templateUid = jsonNode.get("templateUid").asText(); + String templateName = jsonNode.get("templateName").asText(); + String engineType = jsonNode.get("engineType").asText(); + String operator = jsonNode.get("operator").asText(); + + if (StringUtils.isBlank(templateUid)) { + return Message.error("parameters:templateUid can not be empty(请求参数【templateUid】不能为空)"); + } + if (StringUtils.isBlank(templateName)) { + return Message.error("parameters:templateName can not be empty(请求参数【templateName】不能为空)"); + } + if (StringUtils.isBlank(engineType)) { + return Message.error("parameters:engineType can not be empty(请求参数【engineType】不能为空)"); + } + if (StringUtils.isBlank(operator)) { + return Message.error("parameters:operator can not be empty(请求参数【operator】不能为空)"); + } + boolean isFullMode =true; + try { + isFullMode=jsonNode.get("isFullMode").asBoolean(); + logger.info("will update by param isFullMode:"+isFullMode); + } + catch (Exception e) + { + logger.info("will update by default isFullMode:"+isFullMode); + } + + + JsonNode itemParms = jsonNode.get("itemList"); + List confKeyList = new ArrayList<>(); + if (itemParms != null && !itemParms.isNull()) { + try { + confKeyList = + JsonUtils.jackson() + .readValue(itemParms.toString(), new TypeReference>() {}); + } catch (JsonProcessingException e) { + return Message.error("parameters:itemList parsing failed(请求参数【itemList】解析失败)"); + } + } + + + } +} diff --git a/linkis-public-enhancements/linkis-configuration/src/main/java/org/apache/linkis/configuration/service/ConfigKeyService.java b/linkis-public-enhancements/linkis-configuration/src/main/java/org/apache/linkis/configuration/service/ConfigKeyService.java index 665f359483..c5e038a16d 100644 --- a/linkis-public-enhancements/linkis-configuration/src/main/java/org/apache/linkis/configuration/service/ConfigKeyService.java +++ b/linkis-public-enhancements/linkis-configuration/src/main/java/org/apache/linkis/configuration/service/ConfigKeyService.java @@ -17,6 +17,7 @@ package org.apache.linkis.configuration.service; +import org.apache.linkis.configuration.entity.ConfigKey; import org.apache.linkis.configuration.entity.ConfigKeyValue; import org.apache.linkis.configuration.entity.ConfigValue; import org.apache.linkis.configuration.exception.ConfigurationException; @@ -32,6 +33,10 @@ ConfigValue saveConfigValue(ConfigKeyValue configKeyValue, List> labelL List getConfigValue(String configKey, List> labelList) throws ConfigurationException; + + List getConfigKeyList(String engineType) + throws ConfigurationException; + List deleteConfigValue(String configKey, List> labelList) throws ConfigurationException; } diff --git a/linkis-public-enhancements/linkis-configuration/src/main/java/org/apache/linkis/configuration/service/TemplateConfigKeyService.java b/linkis-public-enhancements/linkis-configuration/src/main/java/org/apache/linkis/configuration/service/TemplateConfigKeyService.java new file mode 100644 index 0000000000..3e76601d91 --- /dev/null +++ b/linkis-public-enhancements/linkis-configuration/src/main/java/org/apache/linkis/configuration/service/TemplateConfigKeyService.java @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.configuration.service; + +import org.apache.linkis.configuration.entity.TemplateConfigKeyVo; +import org.apache.linkis.configuration.exception.ConfigurationException; + +import java.util.List; + +public interface TemplateConfigKeyService { + + + Boolean updateKeyMapping(String templateUid, String templateName, String engineType, + String operator, Boolean isFullMod, List itemList) + throws ConfigurationException; +} diff --git a/linkis-public-enhancements/linkis-configuration/src/main/java/org/apache/linkis/configuration/service/impl/ConfigKeyServiceImpl.java b/linkis-public-enhancements/linkis-configuration/src/main/java/org/apache/linkis/configuration/service/impl/ConfigKeyServiceImpl.java index 6811b5e7e2..a2bdfccd0a 100644 --- a/linkis-public-enhancements/linkis-configuration/src/main/java/org/apache/linkis/configuration/service/impl/ConfigKeyServiceImpl.java +++ b/linkis-public-enhancements/linkis-configuration/src/main/java/org/apache/linkis/configuration/service/impl/ConfigKeyServiceImpl.java @@ -163,6 +163,11 @@ public List getConfigValue(String key, List> labelList) return configValues; } + @Override + public List getConfigKeyList(String engineType) throws ConfigurationException { + return configMapper.selectKeyByEngineType(engineType); + } + @Override public List deleteConfigValue(String key, List> labelList) throws ConfigurationException { diff --git a/linkis-public-enhancements/linkis-configuration/src/main/java/org/apache/linkis/configuration/service/impl/TemplateConfigKeyServiceImpl.java b/linkis-public-enhancements/linkis-configuration/src/main/java/org/apache/linkis/configuration/service/impl/TemplateConfigKeyServiceImpl.java new file mode 100644 index 0000000000..72b9ef60d7 --- /dev/null +++ b/linkis-public-enhancements/linkis-configuration/src/main/java/org/apache/linkis/configuration/service/impl/TemplateConfigKeyServiceImpl.java @@ -0,0 +1,139 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.configuration.service.impl; + +import org.apache.linkis.configuration.dao.ConfigMapper; +import org.apache.linkis.configuration.dao.LabelMapper; +import org.apache.linkis.configuration.dao.TemplateConfigKeyMapper; +import org.apache.linkis.configuration.entity.ConfigKey; +import org.apache.linkis.configuration.entity.ConfigKeyValue; +import org.apache.linkis.configuration.entity.TemplateConfigKey; +import org.apache.linkis.configuration.entity.TemplateConfigKeyVo; +import org.apache.linkis.configuration.exception.ConfigurationException; +import org.apache.linkis.configuration.service.ConfigurationService; +import org.apache.linkis.configuration.service.TemplateConfigKeyService; +import org.apache.linkis.configuration.validate.ValidatorManager; +import org.apache.linkis.manager.common.entity.persistence.PersistencerEcNodeInfo; +import org.apache.linkis.manager.label.builder.CombinedLabelBuilder; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +import java.text.MessageFormat; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +@Service +public class TemplateConfigKeyServiceImpl implements TemplateConfigKeyService { + + private static final Logger logger = LoggerFactory.getLogger(TemplateConfigKeyServiceImpl.class); + + @Autowired + private ConfigMapper configMapper; + + @Autowired + private LabelMapper labelMapper; + + @Autowired + private TemplateConfigKeyMapper templateConfigKeyMapper; + + + @Autowired + private ConfigurationService configurationService; + + @Autowired + private ValidatorManager validatorManager; + + + private CombinedLabelBuilder combinedLabelBuilder = new CombinedLabelBuilder(); + + + @Override + public Boolean updateKeyMapping(String templateUid, String templateName, String engineType, + String operator, Boolean isFullMode, List itemList) throws ConfigurationException { + //isFullMode true + //查询对应的数据 并做数据合法性检查 + List keyList=itemList.stream().map(e->e.getKey()).collect(Collectors.toList()); + List configKeyList=configMapper.selectKeyByEngineTypeAndKeyList(engineType,keyList); + //待更新的key id 列表 + List keyIdList=configKeyList.stream().map(e->e.getId()).collect(Collectors.toList()); + if(configKeyList.size()!=itemList.size()) + { + + } + + //组装更新 + List toUpdateOrInsertList=new ArrayList<>(); + + // map k:v---> key：ConfigKey + Map configKeyMap = + configKeyList.stream() + .collect(Collectors.toMap(ConfigKey::getKey, item -> item)); + for (TemplateConfigKeyVo item : itemList) { + + String key=item.getKey(); + ConfigKey temp = configKeyMap.get(item.getKey()); + String validateType=temp.getValidateType(); + String validateRange=temp.getValidateRange(); + String configValue=item.getConfigValue(); + String maxValue=item.getMaxValue(); + + if (!validatorManager + .getOrCreateValidator(validateType) + .validate(configValue, validateRange) + ) { + String msg= MessageFormat.format("Parameter configValue verification failed(参数configValue校验失败):" + + "key:{0}, ValidateType:{1}, ValidateRange:{},ConfigValue:{}", + key,validateType,validateRange,configValue); + throw new ConfigurationException(msg); + } + if (!validatorManager + .getOrCreateValidator(validateType) + .validate(maxValue, validateRange) + ) { + String msg= MessageFormat.format("Parameter maxValue verification failed(参数maxValue校验失败):" + + "key:{0}, ValidateType:{1}, ValidateRange:{},ConfigValue:{}", + key,validateType,validateRange,maxValue); + throw new ConfigurationException(msg); + } + + TemplateConfigKey templateConfigKey=new TemplateConfigKey(); + + + } + //根据不同模式更新数据 + if(isFullMode) + { + //之前在数据库中的数据 需要移除的 + List oldList =templateConfigKeyMapper.selectListByTemplateUuid(templateUid); + List needToRemoveList=oldList.stream().filter(item ->{ + return !keyIdList.contains(item.getKeyId()); + }).map(e->e.getKeyId()).collect(Collectors.toList()); + logger.info("Try to remove old data:["+needToRemoveList+"] for templateUid:"+templateUid); + templateConfigKeyMapper.deleteByTemplateUuidAndKeyIdList(templateUid,needToRemoveList); + } + + + itemList.forEach(); + templateConfigKeyMapper.batchInsertList(templateUid,needToRemoveList); + + } +} diff --git a/linkis-public-enhancements/linkis-configuration/src/main/resources/mapper/common/ConfigKeyLimitForUserMapper.xml b/linkis-public-enhancements/linkis-configuration/src/main/resources/mapper/common/ConfigKeyLimitForUserMapper.xml new file mode 100644 index 0000000000..7c63950613 --- /dev/null +++ b/linkis-public-enhancements/linkis-configuration/src/main/resources/mapper/common/ConfigKeyLimitForUserMapper.xml @@ -0,0 +1,173 @@ + + + + + + + + + + + + + + + + + id, user_name, combined_label_value, key_id, latest_update_template_uuid, is_valid, + create_by, create_time, update_by, update_time + + + + id = #{id,jdbcType=BIGINT} + + + + + insert into config_key_limit_for_user (id, user_name, combined_label_value, + key_id, latest_update_template_uuid, + is_valid, create_by, create_time, update_by, + update_time) + values + + ( + #{item.id,jdbcType=BIGINT}, #{item.userName,jdbcType=VARCHAR}, #{item.combinedLabelValue,jdbcType=VARCHAR}, + #{item.keyId,jdbcType=BIGINT}, #{item.latestUpdateTemplateUuid,jdbcType=VARCHAR}, + #{item.isValid,jdbcType=VARCHAR}, #{item.createBy,jdbcType=VARCHAR}, now(), #{item.updateBy,jdbcType=VARCHAR}, + now()) + + + + delete from config_key_limit_for_user + where id = #{id,jdbcType=BIGINT} + + + insert into config_key_limit_for_user (id, user_name, combined_label_value, + key_id, latest_update_template_uuid, is_valid, + create_by, create_time, update_by, update_time) + values (#{id,jdbcType=BIGINT}, #{userName,jdbcType=VARCHAR}, #{combinedLabelValue,jdbcType=VARCHAR}, + #{keyId,jdbcType=BIGINT}, #{latestUpdateTemplateUuid,jdbcType=VARCHAR}, #{isValid,jdbcType=VARCHAR}, + #{createBy,jdbcType=VARCHAR}, now(), #{updateBy,jdbcType=VARCHAR}, now()) + + + insert into config_key_limit_for_user + + + id, + + + user_name, + + + combined_label_value, + + + key_id, + + + latest_update_template_uuid, + + + is_valid, + + + create_by, + + create_time, + + update_by, + + update_time, + + + + #{id,jdbcType=BIGINT}, + + + #{userName,jdbcType=VARCHAR}, + + + #{combinedLabelValue,jdbcType=VARCHAR}, + + + #{keyId,jdbcType=BIGINT}, + + + #{latestUpdateTemplateUuid,jdbcType=VARCHAR}, + + + #{isValid,jdbcType=VARCHAR}, + + + #{createBy,jdbcType=VARCHAR}, + + now(), + + #{updateBy,jdbcType=VARCHAR}, + + now(), + + + + update config_key_limit_for_user + + + user_name = #{userName,jdbcType=VARCHAR}, + + + combined_label_value = #{combinedLabelValue,jdbcType=VARCHAR}, + + + key_id = #{keyId,jdbcType=BIGINT}, + + + latest_update_template_uuid = #{latestUpdateTemplateUuid,jdbcType=VARCHAR}, + + + is_valid = #{isValid,jdbcType=VARCHAR}, + + + create_by = #{createBy,jdbcType=VARCHAR}, + + + update_by = #{updateBy,jdbcType=VARCHAR}, + + update_time = now(), + + where id = #{id,jdbcType=BIGINT} + + + update config_key_limit_for_user + set user_name = #{userName,jdbcType=VARCHAR}, + combined_label_value = #{combinedLabelValue,jdbcType=VARCHAR}, + key_id = #{keyId,jdbcType=BIGINT}, + latest_update_template_uuid = #{latestUpdateTemplateUuid,jdbcType=VARCHAR}, + is_valid = #{isValid,jdbcType=VARCHAR}, + create_by = #{createBy,jdbcType=VARCHAR}, + update_by = #{updateBy,jdbcType=VARCHAR}, + update_time = now() + where id = #{id,jdbcType=BIGINT} + + + + diff --git a/linkis-public-enhancements/linkis-configuration/src/main/resources/mapper/common/ConfigMapper.xml b/linkis-public-enhancements/linkis-configuration/src/main/resources/mapper/common/ConfigMapper.xml index a5c0af0ef5..efec75659a 100644 --- a/linkis-public-enhancements/linkis-configuration/src/main/resources/mapper/common/ConfigMapper.xml +++ b/linkis-public-enhancements/linkis-configuration/src/main/resources/mapper/common/ConfigMapper.xml @@ -24,7 +24,6 @@ - @@ -32,8 +31,7 @@ - - + @@ -51,8 +49,8 @@ - - + + @@ -137,6 +135,17 @@ WHERE v.config_label_id = #{labelId} + + + + + + + + + + + + + select + + from template_config_key + where id = #{id,jdbcType=BIGINT} + + + id = #{id,jdbcType=BIGINT} + + + + + insert into template_config_key (id, template_name, template_uuid, + key_id, config_value, max_value, + min_value, validate_range, is_valid, + create_by, create_time, update_by, update_time + ) + values + + ( + #{item.id,jdbcType=BIGINT}, #{item.templateName,jdbcType=VARCHAR}, #{item.templateUuid,jdbcType=VARCHAR}, + #{item.keyId,jdbcType=BIGINT}, #{item.configValue,jdbcType=VARCHAR}, #{item.maxValue,jdbcType=VARCHAR}, + #{item.minValue,jdbcType=VARCHAR}, #{item.validateRange,jdbcType=VARCHAR}, #{item.isValid,jdbcType=VARCHAR}, + #{item.createBy,jdbcType=VARCHAR}, now(), #{item.updateBy,jdbcType=VARCHAR}, now() + ) + + + + delete from template_config_key + where id = #{id,jdbcType=BIGINT} + + + insert into template_config_key (id, template_name, template_uuid, + key_id, config_value, max_value, + min_value, validate_range, is_valid, + create_by, create_time, update_by, update_time) + values (#{id,jdbcType=BIGINT}, #{templateName,jdbcType=VARCHAR}, #{templateUuid,jdbcType=VARCHAR}, + #{keyId,jdbcType=BIGINT}, #{configValue,jdbcType=VARCHAR}, #{maxValue,jdbcType=VARCHAR}, + #{minValue,jdbcType=VARCHAR}, #{validateRange,jdbcType=VARCHAR}, #{isValid,jdbcType=VARCHAR}, + #{createBy,jdbcType=VARCHAR}, now(), #{updateBy,jdbcType=VARCHAR}, now()) + + + insert into template_config_key + + + id, + + + template_name, + + + template_uuid, + + + key_id, + + + config_value, + + + max_value, + + + min_value, + + + validate_range, + + + is_valid, + + + create_by, + + create_time, + + update_by, + + update_time, + + + + #{id,jdbcType=BIGINT}, + + + #{templateName,jdbcType=VARCHAR}, + + + #{templateUuid,jdbcType=VARCHAR}, + + + #{keyId,jdbcType=BIGINT}, + + + #{configValue,jdbcType=VARCHAR}, + + + #{maxValue,jdbcType=VARCHAR}, + + + #{minValue,jdbcType=VARCHAR}, + + + #{validateRange,jdbcType=VARCHAR}, + + + #{isValid,jdbcType=VARCHAR}, + + + #{createBy,jdbcType=VARCHAR}, + + now(), + + #{updateBy,jdbcType=VARCHAR}, + + now(), + + + + update template_config_key + + + template_name = #{templateName,jdbcType=VARCHAR}, + + + template_uuid = #{templateUuid,jdbcType=VARCHAR}, + + + key_id = #{keyId,jdbcType=BIGINT}, + + + config_value = #{configValue,jdbcType=VARCHAR}, + + + max_value = #{maxValue,jdbcType=VARCHAR}, + + + min_value = #{minValue,jdbcType=VARCHAR}, + + + validate_range = #{validateRange,jdbcType=VARCHAR}, + + + is_valid = #{isValid,jdbcType=VARCHAR}, + + + create_by = #{createBy,jdbcType=VARCHAR}, + + + update_by = #{updateBy,jdbcType=VARCHAR}, + + update_time = now(), + + where id = #{id,jdbcType=BIGINT} + + + update template_config_key + set template_name = #{templateName,jdbcType=VARCHAR}, + template_uuid = #{templateUuid,jdbcType=VARCHAR}, + key_id = #{keyId,jdbcType=BIGINT}, + config_value = #{configValue,jdbcType=VARCHAR}, + max_value = #{maxValue,jdbcType=VARCHAR}, + min_value = #{minValue,jdbcType=VARCHAR}, + validate_range = #{validateRange,jdbcType=VARCHAR}, + is_valid = #{isValid,jdbcType=VARCHAR}, + create_by = #{createBy,jdbcType=VARCHAR}, + update_by = #{updateBy,jdbcType=VARCHAR}, + update_time = now() + where id = #{id,jdbcType=BIGINT} + + + + + + + + delete + from template_config_key + where template_uuid = #{templateUuid,jdbcType=VARCHAR} + and key_id in + + #{item} + + + diff --git a/linkis-public-enhancements/linkis-configuration/src/main/scala/org/apache/linkis/configuration/service/ConfigurationService.scala b/linkis-public-enhancements/linkis-configuration/src/main/scala/org/apache/linkis/configuration/service/ConfigurationService.scala index 9b5f237021..718a97cfb2 100644 --- a/linkis-public-enhancements/linkis-configuration/src/main/scala/org/apache/linkis/configuration/service/ConfigurationService.scala +++ b/linkis-public-enhancements/linkis-configuration/src/main/scala/org/apache/linkis/configuration/service/ConfigurationService.scala @@ -93,11 +93,6 @@ class ConfigurationService extends Logging { } } - def insertCreator(creator: String): Unit = { - val creatorID: Long = configMapper.selectAppIDByAppName(creator) - if (creatorID > 0) configMapper.insertCreator(creator) - else logger.warn(s"creator${creator} exists") - } def checkAndCreateUserLabel( settings: util.List[ConfigKeyValue], diff --git a/linkis-public-enhancements/linkis-configuration/src/test/java/org/apache/linkis/configuration/dao/ConfigMapperTest.java b/linkis-public-enhancements/linkis-configuration/src/test/java/org/apache/linkis/configuration/dao/ConfigMapperTest.java index c636b91359..d8b37bed1d 100644 --- a/linkis-public-enhancements/linkis-configuration/src/test/java/org/apache/linkis/configuration/dao/ConfigMapperTest.java +++ b/linkis-public-enhancements/linkis-configuration/src/test/java/org/apache/linkis/configuration/dao/ConfigMapperTest.java @@ -154,12 +154,6 @@ void testListKeyByStringValue() { // assertEquals(7, configKeyList.size()); } - @Test - void testInsertCreator() { - // mapper方法没有对应的实现类 - // configMapper.insertCreator("tom"); - } - @Test void testGetCategory() { List categoryLabelList = configMapper.getCategory(); From 3eb90414f73f853c6ffa2acfd085b2b3f5eee0e6 Mon Sep 17 00:00:00 2001 From: casionone Date: Tue, 20 Jun 2023 00:06:58 +0800 Subject: [PATCH 170/261] feature add config template --- .../dao/ConfigKeyLimitForUserMapper.java | 186 ++--- .../configuration/dao/ConfigMapper.java | 9 +- .../linkis/configuration/dao/LabelMapper.java | 5 + .../dao/TemplateConfigKeyMapper.java | 197 ++--- .../entity/ConfigKeyLimitForUser.java | 569 +++++++------- .../entity/TemplateConfigKey.java | 722 +++++++++--------- .../entity/TemplateConfigKeyVo.java | 115 +-- .../restful/api/ConfigurationRestfulApi.java | 15 +- .../api/ConfigurationTemplateRestfulApi.java | 238 ++++-- .../service/ConfigKeyService.java | 4 +- .../service/TemplateConfigKeyService.java | 22 +- .../impl/TemplateConfigKeyServiceImpl.java | 351 +++++++-- .../common/ConfigKeyLimitForUserMapper.xml | 53 ++ .../resources/mapper/common/ConfigMapper.xml | 17 +- .../resources/mapper/common/LabelMapper.xml | 21 + .../mapper/common/TemplateConfigKeyMapper.xml | 67 ++ 16 files changed, 1538 insertions(+), 1053 deletions(-) diff --git a/linkis-public-enhancements/linkis-configuration/src/main/java/org/apache/linkis/configuration/dao/ConfigKeyLimitForUserMapper.java b/linkis-public-enhancements/linkis-configuration/src/main/java/org/apache/linkis/configuration/dao/ConfigKeyLimitForUserMapper.java index 2e7626a194..c9f259d661 100644 --- a/linkis-public-enhancements/linkis-configuration/src/main/java/org/apache/linkis/configuration/dao/ConfigKeyLimitForUserMapper.java +++ b/linkis-public-enhancements/linkis-configuration/src/main/java/org/apache/linkis/configuration/dao/ConfigKeyLimitForUserMapper.java @@ -1,100 +1,108 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.apache.linkis.configuration.dao; -import java.util.List; -import org.apache.ibatis.annotations.Param; import org.apache.linkis.configuration.entity.ConfigKeyLimitForUser; +import org.apache.ibatis.annotations.Param; + +import java.util.List; /** - * config_key_limit_for_user表的dao接口类 - * @Description + * config_key_limit_for_user表的dao接口类 @Description + * * @version 1.0 - * @author webank */ public interface ConfigKeyLimitForUserMapper { - /** - * 根据主键删除数据库的记录 - * - * @param id - */ - int deleteByPrimaryKey(Long id); - - /** - * 新写入数据库记录 - * - * @param configKeyLimitForUser - */ - int insert(ConfigKeyLimitForUser configKeyLimitForUser); - - /** - * 批量插入记录，建议List一次不要超过1000条 - * - * @param list - */ - int batchInsertList(List list); - - /** - * 动态字段,写入数据库记录 - * - * @param configKeyLimitForUser - */ - int insertSelective(ConfigKeyLimitForUser configKeyLimitForUser); - - /** - * 根据指定主键获取一条数据库记录 - * - * @param id - */ - ConfigKeyLimitForUser selectByPrimaryKey(Long id); - - /** - * 查询分页数据条数 - 示例方法 - * - * @param id - */ - int selectCountByPage(Long id); - - /** - * 查询分页数据列表 - 示例方法 - * public DataPage selectByPage(Id id, int pageNo, int pageSize) { - * if (pageNo > 100) { - * pageNo = 100; - * } - * if (pageNo < 1) { - * pageNo = 1; - * } - * if (pageSize > 50) { - * pageSize = 50; - * } - * if (pageSize < 1) { - * pageSize = 1; - * } - * int totalCount = configKeyLimitForUserDAO.selectCountByPage(id); - * List list = configKeyLimitForUserDAO.selectListByPage(id, pageNo, pageSize); - * DataPage dp = new DataPage<>(list, pageSize, pageNo, totalCount); - * return dp; - * } - * - * @param id - * @param pageNo - * @param pageSize - */ - List selectListByPage(@Param("id") Long id, @Param("pageNo") int pageNo, @Param("pageSize") int pageSize); - - /** - * 动态字段,根据主键来更新符合条件的数据库记录 - * - * @param configKeyLimitForUser - */ - int updateByPrimaryKeySelective(ConfigKeyLimitForUser configKeyLimitForUser); - - /** - * 根据主键来更新符合条件的数据库记录 - * - * @param configKeyLimitForUser - */ - int updateByPrimaryKey(ConfigKeyLimitForUser configKeyLimitForUser); - - // === 下方为用户自定义模块,下次生成会保留 === + /** + * 根据主键删除数据库的记录 + * + * @param id + */ + int deleteByPrimaryKey(Long id); + + /** + * 新写入数据库记录 + * + * @param configKeyLimitForUser + */ + int insert(ConfigKeyLimitForUser configKeyLimitForUser); + + /** + * 批量插入记录，建议List一次不要超过1000条 + * + * @param list + */ + int batchInsertList(List list); + + /** + * 动态字段,写入数据库记录 + * + * @param configKeyLimitForUser + */ + int insertSelective(ConfigKeyLimitForUser configKeyLimitForUser); + + /** + * 根据指定主键获取一条数据库记录 + * + * @param id + */ + ConfigKeyLimitForUser selectByPrimaryKey(Long id); + + /** + * 查询分页数据条数 - 示例方法 + * + * @param id + */ + int selectCountByPage(Long id); + + /** + * 查询分页数据列表 - 示例方法 public DataPage selectByPage(Id id, int pageNo, int + * pageSize) { if (pageNo > 100) { pageNo = 100; } if (pageNo < 1) { pageNo = 1; } if (pageSize > + * 50) { pageSize = 50; } if (pageSize < 1) { pageSize = 1; } int totalCount = + * configKeyLimitForUserDAO.selectCountByPage(id); List list = + * configKeyLimitForUserDAO.selectListByPage(id, pageNo, pageSize); + * DataPage dp = new DataPage<>(list, pageSize, pageNo, totalCount); return + * dp; } + * + * @param id + * @param pageNo + * @param pageSize + */ + List selectListByPage( + @Param("id") Long id, @Param("pageNo") int pageNo, @Param("pageSize") int pageSize); + + /** + * 动态字段,根据主键来更新符合条件的数据库记录 + * + * @param configKeyLimitForUser + */ + int updateByPrimaryKeySelective(ConfigKeyLimitForUser configKeyLimitForUser); + + /** + * 根据主键来更新符合条件的数据库记录 + * + * @param configKeyLimitForUser + */ + int updateByPrimaryKey(ConfigKeyLimitForUser configKeyLimitForUser); + + // === 下方为用户自定义模块,下次生成会保留 === + + int batchInsertOrUpdateList(List list); } diff --git a/linkis-public-enhancements/linkis-configuration/src/main/java/org/apache/linkis/configuration/dao/ConfigMapper.java b/linkis-public-enhancements/linkis-configuration/src/main/java/org/apache/linkis/configuration/dao/ConfigMapper.java index cb9c3e0f97..937b343389 100644 --- a/linkis-public-enhancements/linkis-configuration/src/main/java/org/apache/linkis/configuration/dao/ConfigMapper.java +++ b/linkis-public-enhancements/linkis-configuration/src/main/java/org/apache/linkis/configuration/dao/ConfigMapper.java @@ -28,13 +28,14 @@ public interface ConfigMapper { - List getConfigKeyByLabelIds(@Param("ids") List ids); List getConfigKeyValueByLabelId(@Param("labelId") Integer labelId); void insertValue(ConfigValue configValue); + void batchInsertValue(@Param("configValueList") List configValueList); + ConfigValue getConfigValueById(@Param("id") Long id); ConfigValue getConfigValueByKeyAndLabel(ConfigValue configValue); @@ -53,13 +54,13 @@ public interface ConfigMapper { List selectKeyByEngineType(@Param("engineType") String engineType); - List selectKeyByEngineTypeAndKeyList(@Param("engineType") String engineType, - @Param("KeyList") List keyList); + List selectKeyByEngineTypeAndKeyList( + @Param("engineType") String engineType, @Param("KeyList") List keyList); + List selectKeyByKeyIdList(@Param("KeyIdList") List keyList); List listKeyByStringValue(@Param("stringValue") String stringValue); - List getCategory(); CategoryLabel getCategoryById(@Param("id") Integer id); diff --git a/linkis-public-enhancements/linkis-configuration/src/main/java/org/apache/linkis/configuration/dao/LabelMapper.java b/linkis-public-enhancements/linkis-configuration/src/main/java/org/apache/linkis/configuration/dao/LabelMapper.java index d199134b4b..1a513e3352 100644 --- a/linkis-public-enhancements/linkis-configuration/src/main/java/org/apache/linkis/configuration/dao/LabelMapper.java +++ b/linkis-public-enhancements/linkis-configuration/src/main/java/org/apache/linkis/configuration/dao/LabelMapper.java @@ -28,8 +28,13 @@ public interface LabelMapper { ConfigLabel getLabelByKeyValue( @Param("labelKey") String labelKey, @Param("stringValue") String stringValue); + // label key:combined_userCreator_engineType + List selectUserCreatorEngineTypeLabelList(@Param("itemList") List itemList); + void insertLabel(ConfigLabel label); + void batchInsertLabel(@Param("labelList") List labelList); + void deleteLabel(@Param("ids") List ids); ConfigLabel getLabelById(@Param("id") Integer id); diff --git a/linkis-public-enhancements/linkis-configuration/src/main/java/org/apache/linkis/configuration/dao/TemplateConfigKeyMapper.java b/linkis-public-enhancements/linkis-configuration/src/main/java/org/apache/linkis/configuration/dao/TemplateConfigKeyMapper.java index 1e5967871d..cec10fbc24 100644 --- a/linkis-public-enhancements/linkis-configuration/src/main/java/org/apache/linkis/configuration/dao/TemplateConfigKeyMapper.java +++ b/linkis-public-enhancements/linkis-configuration/src/main/java/org/apache/linkis/configuration/dao/TemplateConfigKeyMapper.java @@ -1,103 +1,114 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.apache.linkis.configuration.dao; -import java.util.List; -import org.apache.ibatis.annotations.Param; import org.apache.linkis.configuration.entity.TemplateConfigKey; +import org.apache.ibatis.annotations.Param; + +import java.util.List; + /** - * template_config_key表的dao接口类 - * @Description + * template_config_key表的dao接口类 @Description + * * @version 1.0 - * @author webank */ public interface TemplateConfigKeyMapper { - /** - * 根据主键删除数据库的记录 - * - * @param id - */ - int deleteByPrimaryKey(Long id); - - /** - * 新写入数据库记录 - * - * @param templateConfigKey - */ - int insert(TemplateConfigKey templateConfigKey); - - /** - * 批量插入记录，建议List一次不要超过1000条 - * - * @param list - */ - int batchInsertList(List list); - - /** - * 动态字段,写入数据库记录 - * - * @param templateConfigKey - */ - int insertSelective(TemplateConfigKey templateConfigKey); - - /** - * 根据指定主键获取一条数据库记录 - * - * @param id - */ - TemplateConfigKey selectByPrimaryKey(Long id); - - /** - * 查询分页数据条数 - 示例方法 - * - * @param id - */ - int selectCountByPage(Long id); - - /** - * 查询分页数据列表 - 示例方法 - * public DataPage selectByPage(Id id, int pageNo, int pageSize) { - * if (pageNo > 100) { - * pageNo = 100; - * } - * if (pageNo < 1) { - * pageNo = 1; - * } - * if (pageSize > 50) { - * pageSize = 50; - * } - * if (pageSize < 1) { - * pageSize = 1; - * } - * int totalCount = templateConfigKeyDAO.selectCountByPage(id); - * List list = templateConfigKeyDAO.selectListByPage(id, pageNo, pageSize); - * DataPage dp = new DataPage<>(list, pageSize, pageNo, totalCount); - * return dp; - * } - * - * @param id - * @param pageNo - * @param pageSize - */ - List selectListByPage(@Param("id") Long id, @Param("pageNo") int pageNo, @Param("pageSize") int pageSize); - - /** - * 动态字段,根据主键来更新符合条件的数据库记录 - * - * @param templateConfigKey - */ - int updateByPrimaryKeySelective(TemplateConfigKey templateConfigKey); - - /** - * 根据主键来更新符合条件的数据库记录 - * - * @param templateConfigKey - */ - int updateByPrimaryKey(TemplateConfigKey templateConfigKey); - - // === 下方为用户自定义模块,下次生成会保留 === - List selectListByTemplateUuid(@Param("templateUuid") String templateUuid); - - - int deleteByTemplateUuidAndKeyIdList(@Param("templateUuid") String templateUuid, @Param("KeyIdList")List KeyIdList); + /** + * 根据主键删除数据库的记录 + * + * @param id + */ + int deleteByPrimaryKey(Long id); + + /** + * 新写入数据库记录 + * + * @param templateConfigKey + */ + int insert(TemplateConfigKey templateConfigKey); + + /** + * 批量插入记录，建议List一次不要超过1000条 + * + * @param list + */ + int batchInsertList(List list); + + /** + * 动态字段,写入数据库记录 + * + * @param templateConfigKey + */ + int insertSelective(TemplateConfigKey templateConfigKey); + + /** + * 根据指定主键获取一条数据库记录 + * + * @param id + */ + TemplateConfigKey selectByPrimaryKey(Long id); + + /** + * 查询分页数据条数 - 示例方法 + * + * @param id + */ + int selectCountByPage(Long id); + + /** + * 查询分页数据列表 - 示例方法 public DataPage selectByPage(Id id, int pageNo, int + * pageSize) { if (pageNo > 100) { pageNo = 100; } if (pageNo < 1) { pageNo = 1; } if (pageSize > + * 50) { pageSize = 50; } if (pageSize < 1) { pageSize = 1; } int totalCount = + * templateConfigKeyDAO.selectCountByPage(id); List list = + * templateConfigKeyDAO.selectListByPage(id, pageNo, pageSize); DataPage dp = + * new DataPage<>(list, pageSize, pageNo, totalCount); return dp; } + * + * @param id + * @param pageNo + * @param pageSize + */ + List selectListByPage( + @Param("id") Long id, @Param("pageNo") int pageNo, @Param("pageSize") int pageSize); + + /** + * 动态字段,根据主键来更新符合条件的数据库记录 + * + * @param templateConfigKey + */ + int updateByPrimaryKeySelective(TemplateConfigKey templateConfigKey); + + /** + * 根据主键来更新符合条件的数据库记录 + * + * @param templateConfigKey + */ + int updateByPrimaryKey(TemplateConfigKey templateConfigKey); + + // === 下方为用户自定义模块,下次生成会保留 === + List selectListByTemplateUuid(@Param("templateUuid") String templateUuid); + + int deleteByTemplateUuidAndKeyIdList( + @Param("templateUuid") String templateUuid, @Param("KeyIdList") List KeyIdList); + + int batchInsertOrUpdateList(List list); + + List selectListByTemplateUuidList( + @Param("templateUuidList") List templateUuidList); } diff --git a/linkis-public-enhancements/linkis-configuration/src/main/java/org/apache/linkis/configuration/entity/ConfigKeyLimitForUser.java b/linkis-public-enhancements/linkis-configuration/src/main/java/org/apache/linkis/configuration/entity/ConfigKeyLimitForUser.java index c3685e4902..8fd64dff7e 100644 --- a/linkis-public-enhancements/linkis-configuration/src/main/java/org/apache/linkis/configuration/entity/ConfigKeyLimitForUser.java +++ b/linkis-public-enhancements/linkis-configuration/src/main/java/org/apache/linkis/configuration/entity/ConfigKeyLimitForUser.java @@ -1,10 +1,27 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.apache.linkis.configuration.entity; import java.util.Date; /** - * config_key_limit_for_user表的entity类 - * @Description + * config_key_limit_for_user表的entity类 @Description + * * @version 1.0 * @author webank * @see http://10.107.99.84:53661/wego-rad/site/plugin/weup-tool/mybatisGen.html @@ -12,296 +29,258 @@ */ public class ConfigKeyLimitForUser { - /** - * 表字段 : id - * 字段类型 : bigint(19) - */ - private Long id; - - /** - * 用户名 - * 表字段 : user_name - * 字段类型 : varchar(50) - */ - private String userName; - - /** - * 组合标签 combined_userCreator_engineType 如 hadoop-IDE,spark-2.4.3 - * 表字段 : combined_label_value - * 字段类型 : varchar(200) - */ - private String combinedLabelValue; - - /** - * id of linkis_ps_configuration_config_key - * 表字段 : key_id - * 字段类型 : bigint(19) - */ - private Long keyId; - - /** - * uuid 第三方侧记录的模板id - * 表字段 : latest_update_template_uuid - * 字段类型 : varchar(34) - */ - private String latestUpdateTemplateUuid; - - /** - * 是否有效 预留 Y/N - * 表字段 : is_valid - * 字段类型 : varchar(2) - */ - private String isValid; - - /** - * 创建人 - * 表字段 : create_by - * 字段类型 : varchar(50) - */ - private String createBy; - - /** - * create time - * 表字段 : create_time - * 字段类型 : timestamp(19) - * 默认值 : CURRENT_TIMESTAMP - */ - private Date createTime; - - /** - * 更新人 - * 表字段 : update_by - * 字段类型 : varchar(50) - */ - private String updateBy; - - /** - * update time - * 表字段 : update_time - * 字段类型 : timestamp(19) - * 默认值 : CURRENT_TIMESTAMP - */ - private Date updateTime; - - /** - * 获取：
- * 字段：id - * - * @return - */ - public Long getId() { - return id; - } - - /** - * 设置：
- * 字段：id - * - * @param - */ - public void setId(Long id) { - this.id = id; - } - - /** - * 获取：用户名
- * 字段：user_name - * - * @return - */ - public String getUserName() { - return userName; - } - - /** - * 设置：用户名
- * 字段：user_name - * - * @param - */ - public void setUserName(String userName) { - this.userName = userName; - } - - /** - * 获取：组合标签 combined_userCreator_engineType 如 hadoop-IDE,spark-2.4.3
- * 字段：combined_label_value - * - * @return - */ - public String getCombinedLabelValue() { - return combinedLabelValue; - } - - /** - * 设置：组合标签 combined_userCreator_engineType 如 hadoop-IDE,spark-2.4.3
- * 字段：combined_label_value - * - * @param - */ - public void setCombinedLabelValue(String combinedLabelValue) { - this.combinedLabelValue = combinedLabelValue; - } - - /** - * 获取：id of linkis_ps_configuration_config_key
- * 字段：key_id - * - * @return - */ - public Long getKeyId() { - return keyId; - } - - /** - * 设置：id of linkis_ps_configuration_config_key
- * 字段：key_id - * - * @param - */ - public void setKeyId(Long keyId) { - this.keyId = keyId; - } - - /** - * 获取：uuid 第三方侧记录的模板id
- * 字段：latest_update_template_uuid - * - * @return - */ - public String getLatestUpdateTemplateUuid() { - return latestUpdateTemplateUuid; - } - - /** - * 设置：uuid 第三方侧记录的模板id
- * 字段：latest_update_template_uuid - * - * @param - */ - public void setLatestUpdateTemplateUuid(String latestUpdateTemplateUuid) { - this.latestUpdateTemplateUuid = latestUpdateTemplateUuid; - } - - /** - * 获取：是否有效 预留 Y/N
- * 字段：is_valid - * - * @return - */ - public String getIsValid() { - return isValid; - } - - /** - * 设置：是否有效 预留 Y/N
- * 字段：is_valid - * - * @param - */ - public void setIsValid(String isValid) { - this.isValid = isValid; - } - - /** - * 获取：创建人
- * 字段：create_by - * - * @return - */ - public String getCreateBy() { - return createBy; - } - - /** - * 设置：创建人
- * 字段：create_by - * - * @param - */ - public void setCreateBy(String createBy) { - this.createBy = createBy; - } - - /** - * 获取：create time
- * 字段：create_time - * - * @return - */ - public Date getCreateTime() { - return createTime; - } - - /** - * 设置：create time
- * 字段：create_time - * - * @param - */ - public void setCreateTime(Date createTime) { - this.createTime = createTime; - } - - /** - * 获取：更新人
- * 字段：update_by - * - * @return - */ - public String getUpdateBy() { - return updateBy; - } - - /** - * 设置：更新人
- * 字段：update_by - * - * @param - */ - public void setUpdateBy(String updateBy) { - this.updateBy = updateBy; - } - - /** - * 获取：update time
- * 字段：update_time - * - * @return - */ - public Date getUpdateTime() { - return updateTime; - } - - /** - * 设置：update time
- * 字段：update_time - * - * @param - */ - public void setUpdateTime(Date updateTime) { - this.updateTime = updateTime; - } - - @Override - public String toString() { - StringBuilder sb = new StringBuilder(); - sb.append(getClass().getSimpleName()); - sb.append(" ["); - sb.append("Hash = ").append(hashCode()); - sb.append(", id=").append(id); - sb.append(", userName=").append(userName); - sb.append(", combinedLabelValue=").append(combinedLabelValue); - sb.append(", keyId=").append(keyId); - sb.append(", latestUpdateTemplateUuid=").append(latestUpdateTemplateUuid); - sb.append(", isValid=").append(isValid); - sb.append(", createBy=").append(createBy); - sb.append(", createTime=").append(createTime); - sb.append(", updateBy=").append(updateBy); - sb.append(", updateTime=").append(updateTime); - sb.append(']'); - return sb.toString(); - } - - // === 下方为用户自定义模块,下次生成会保留 === -} \ No newline at end of file + /** 表字段 : id 字段类型 : bigint(19) */ + private Long id; + + /** 用户名 表字段 : user_name 字段类型 : varchar(50) */ + private String userName; + + /** + * 组合标签 combined_userCreator_engineType 如 hadoop-IDE,spark-2.4.3 表字段 : combined_label_value 字段类型 : + * varchar(200) + */ + private String combinedLabelValue; + + /** id of linkis_ps_configuration_config_key 表字段 : key_id 字段类型 : bigint(19) */ + private Long keyId; + + /** uuid 第三方侧记录的模板id 表字段 : latest_update_template_uuid 字段类型 : varchar(34) */ + private String latestUpdateTemplateUuid; + + /** 是否有效 预留 Y/N 表字段 : is_valid 字段类型 : varchar(2) */ + private String isValid; + + /** 创建人 表字段 : create_by 字段类型 : varchar(50) */ + private String createBy; + + /** create time 表字段 : create_time 字段类型 : timestamp(19) 默认值 : CURRENT_TIMESTAMP */ + private Date createTime; + + /** 更新人 表字段 : update_by 字段类型 : varchar(50) */ + private String updateBy; + + /** update time 表字段 : update_time 字段类型 : timestamp(19) 默认值 : CURRENT_TIMESTAMP */ + private Date updateTime; + + /** + * 获取：
+ * 字段：id + * + * @return + */ + public Long getId() { + return id; + } + + /** + * 设置：
+ * 字段：id + * + * @param + */ + public void setId(Long id) { + this.id = id; + } + + /** + * 获取：用户名
+ * 字段：user_name + * + * @return + */ + public String getUserName() { + return userName; + } + + /** + * 设置：用户名
+ * 字段：user_name + * + * @param + */ + public void setUserName(String userName) { + this.userName = userName; + } + + /** + * 获取：组合标签 combined_userCreator_engineType 如 hadoop-IDE,spark-2.4.3
+ * 字段：combined_label_value + * + * @return + */ + public String getCombinedLabelValue() { + return combinedLabelValue; + } + + /** + * 设置：组合标签 combined_userCreator_engineType 如 hadoop-IDE,spark-2.4.3
+ * 字段：combined_label_value + * + * @param + */ + public void setCombinedLabelValue(String combinedLabelValue) { + this.combinedLabelValue = combinedLabelValue; + } + + /** + * 获取：id of linkis_ps_configuration_config_key
+ * 字段：key_id + * + * @return + */ + public Long getKeyId() { + return keyId; + } + + /** + * 设置：id of linkis_ps_configuration_config_key
+ * 字段：key_id + * + * @param + */ + public void setKeyId(Long keyId) { + this.keyId = keyId; + } + + /** + * 获取：uuid 第三方侧记录的模板id
+ * 字段：latest_update_template_uuid + * + * @return + */ + public String getLatestUpdateTemplateUuid() { + return latestUpdateTemplateUuid; + } + + /** + * 设置：uuid 第三方侧记录的模板id
+ * 字段：latest_update_template_uuid + * + * @param + */ + public void setLatestUpdateTemplateUuid(String latestUpdateTemplateUuid) { + this.latestUpdateTemplateUuid = latestUpdateTemplateUuid; + } + + /** + * 获取：是否有效 预留 Y/N
+ * 字段：is_valid + * + * @return + */ + public String getIsValid() { + return isValid; + } + + /** + * 设置：是否有效 预留 Y/N
+ * 字段：is_valid + * + * @param + */ + public void setIsValid(String isValid) { + this.isValid = isValid; + } + + /** + * 获取：创建人
+ * 字段：create_by + * + * @return + */ + public String getCreateBy() { + return createBy; + } + + /** + * 设置：创建人
+ * 字段：create_by + * + * @param + */ + public void setCreateBy(String createBy) { + this.createBy = createBy; + } + + /** + * 获取：create time
+ * 字段：create_time + * + * @return + */ + public Date getCreateTime() { + return createTime; + } + + /** + * 设置：create time
+ * 字段：create_time + * + * @param + */ + public void setCreateTime(Date createTime) { + this.createTime = createTime; + } + + /** + * 获取：更新人
+ * 字段：update_by + * + * @return + */ + public String getUpdateBy() { + return updateBy; + } + + /** + * 设置：更新人
+ * 字段：update_by + * + * @param + */ + public void setUpdateBy(String updateBy) { + this.updateBy = updateBy; + } + + /** + * 获取：update time
+ * 字段：update_time + * + * @return + */ + public Date getUpdateTime() { + return updateTime; + } + + /** + * 设置：update time
+ * 字段：update_time + * + * @param + */ + public void setUpdateTime(Date updateTime) { + this.updateTime = updateTime; + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + sb.append(getClass().getSimpleName()); + sb.append(" ["); + sb.append("Hash = ").append(hashCode()); + sb.append(", id=").append(id); + sb.append(", userName=").append(userName); + sb.append(", combinedLabelValue=").append(combinedLabelValue); + sb.append(", keyId=").append(keyId); + sb.append(", latestUpdateTemplateUuid=").append(latestUpdateTemplateUuid); + sb.append(", isValid=").append(isValid); + sb.append(", createBy=").append(createBy); + sb.append(", createTime=").append(createTime); + sb.append(", updateBy=").append(updateBy); + sb.append(", updateTime=").append(updateTime); + sb.append(']'); + return sb.toString(); + } + + // === 下方为用户自定义模块,下次生成会保留 === +} diff --git a/linkis-public-enhancements/linkis-configuration/src/main/java/org/apache/linkis/configuration/entity/TemplateConfigKey.java b/linkis-public-enhancements/linkis-configuration/src/main/java/org/apache/linkis/configuration/entity/TemplateConfigKey.java index 3df6532037..a89dbd5fa7 100644 --- a/linkis-public-enhancements/linkis-configuration/src/main/java/org/apache/linkis/configuration/entity/TemplateConfigKey.java +++ b/linkis-public-enhancements/linkis-configuration/src/main/java/org/apache/linkis/configuration/entity/TemplateConfigKey.java @@ -1,10 +1,27 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.apache.linkis.configuration.entity; import java.util.Date; /** - * template_config_key表的entity类 - * @Description + * template_config_key表的entity类 @Description + * * @version 1.0 * @author webank * @see http://10.107.99.84:53661/wego-rad/site/plugin/weup-tool/mybatisGen.html @@ -12,380 +29,327 @@ */ public class TemplateConfigKey { - /** - * 表字段 : id - * 字段类型 : bigint(19) - */ - private Long id; - - /** - * 配置模板名称 冗余存储 - * 表字段 : template_name - * 字段类型 : varchar(200) - */ - private String templateName; - - /** - * uuid 第三方侧记录的模板id - * 表字段 : template_uuid - * 字段类型 : varchar(34) - */ - private String templateUuid; - - /** - * id of linkis_ps_configuration_config_key - * 表字段 : key_id - * 字段类型 : bigint(19) - */ - private Long keyId; - - /** - * 配置值 - * 表字段 : config_value - * 字段类型 : varchar(200) - */ - private String configValue; - - /** - * 上限值 - * 表字段 : max_value - * 字段类型 : varchar(50) - */ - private String maxValue; - - /** - * 下限值（预留） - * 表字段 : min_value - * 字段类型 : varchar(50) - */ - private String minValue; - - /** - * 校验正则(预留) - * 表字段 : validate_range - * 字段类型 : varchar(50) - */ - private String validateRange; - - /** - * 是否有效 预留 Y/N - * 表字段 : is_valid - * 字段类型 : varchar(2) - */ - private String isValid; - - /** - * 创建人 - * 表字段 : create_by - * 字段类型 : varchar(50) - */ - private String createBy; - - /** - * create time - * 表字段 : create_time - * 字段类型 : timestamp(19) - * 默认值 : CURRENT_TIMESTAMP - */ - private Date createTime; - - /** - * 更新人 - * 表字段 : update_by - * 字段类型 : varchar(50) - */ - private String updateBy; - - /** - * update time - * 表字段 : update_time - * 字段类型 : timestamp(19) - * 默认值 : CURRENT_TIMESTAMP - */ - private Date updateTime; - - /** - * 获取：
- * 字段：id - * - * @return - */ - public Long getId() { - return id; - } - - /** - * 设置：
- * 字段：id - * - * @param - */ - public void setId(Long id) { - this.id = id; - } - - /** - * 获取：配置模板名称 冗余存储
- * 字段：template_name - * - * @return - */ - public String getTemplateName() { - return templateName; - } - - /** - * 设置：配置模板名称 冗余存储
- * 字段：template_name - * - * @param - */ - public void setTemplateName(String templateName) { - this.templateName = templateName; - } - - /** - * 获取：uuid 第三方侧记录的模板id
- * 字段：template_uuid - * - * @return - */ - public String getTemplateUuid() { - return templateUuid; - } - - /** - * 设置：uuid 第三方侧记录的模板id
- * 字段：template_uuid - * - * @param - */ - public void setTemplateUuid(String templateUuid) { - this.templateUuid = templateUuid; - } - - /** - * 获取：id of linkis_ps_configuration_config_key
- * 字段：key_id - * - * @return - */ - public Long getKeyId() { - return keyId; - } - - /** - * 设置：id of linkis_ps_configuration_config_key
- * 字段：key_id - * - * @param - */ - public void setKeyId(Long keyId) { - this.keyId = keyId; - } - - /** - * 获取：配置值
- * 字段：config_value - * - * @return - */ - public String getConfigValue() { - return configValue; - } - - /** - * 设置：配置值
- * 字段：config_value - * - * @param - */ - public void setConfigValue(String configValue) { - this.configValue = configValue; - } - - /** - * 获取：上限值
- * 字段：max_value - * - * @return - */ - public String getMaxValue() { - return maxValue; - } - - /** - * 设置：上限值
- * 字段：max_value - * - * @param - */ - public void setMaxValue(String maxValue) { - this.maxValue = maxValue; - } - - /** - * 获取：下限值（预留）
- * 字段：min_value - * - * @return - */ - public String getMinValue() { - return minValue; - } - - /** - * 设置：下限值（预留）
- * 字段：min_value - * - * @param - */ - public void setMinValue(String minValue) { - this.minValue = minValue; - } - - /** - * 获取：校验正则(预留)
- * 字段：validate_range - * - * @return - */ - public String getValidateRange() { - return validateRange; - } - - /** - * 设置：校验正则(预留)
- * 字段：validate_range - * - * @param - */ - public void setValidateRange(String validateRange) { - this.validateRange = validateRange; - } - - /** - * 获取：是否有效 预留 Y/N
- * 字段：is_valid - * - * @return - */ - public String getIsValid() { - return isValid; - } - - /** - * 设置：是否有效 预留 Y/N
- * 字段：is_valid - * - * @param - */ - public void setIsValid(String isValid) { - this.isValid = isValid; - } - - /** - * 获取：创建人
- * 字段：create_by - * - * @return - */ - public String getCreateBy() { - return createBy; - } - - /** - * 设置：创建人
- * 字段：create_by - * - * @param - */ - public void setCreateBy(String createBy) { - this.createBy = createBy; - } - - /** - * 获取：create time
- * 字段：create_time - * - * @return - */ - public Date getCreateTime() { - return createTime; - } - - /** - * 设置：create time
- * 字段：create_time - * - * @param - */ - public void setCreateTime(Date createTime) { - this.createTime = createTime; - } - - /** - * 获取：更新人
- * 字段：update_by - * - * @return - */ - public String getUpdateBy() { - return updateBy; - } - - /** - * 设置：更新人
- * 字段：update_by - * - * @param - */ - public void setUpdateBy(String updateBy) { - this.updateBy = updateBy; - } - - /** - * 获取：update time
- * 字段：update_time - * - * @return - */ - public Date getUpdateTime() { - return updateTime; - } - - /** - * 设置：update time
- * 字段：update_time - * - * @param - */ - public void setUpdateTime(Date updateTime) { - this.updateTime = updateTime; - } - - @Override - public String toString() { - StringBuilder sb = new StringBuilder(); - sb.append(getClass().getSimpleName()); - sb.append(" ["); - sb.append("Hash = ").append(hashCode()); - sb.append(", id=").append(id); - sb.append(", templateName=").append(templateName); - sb.append(", templateUuid=").append(templateUuid); - sb.append(", keyId=").append(keyId); - sb.append(", configValue=").append(configValue); - sb.append(", maxValue=").append(maxValue); - sb.append(", minValue=").append(minValue); - sb.append(", validateRange=").append(validateRange); - sb.append(", isValid=").append(isValid); - sb.append(", createBy=").append(createBy); - sb.append(", createTime=").append(createTime); - sb.append(", updateBy=").append(updateBy); - sb.append(", updateTime=").append(updateTime); - sb.append(']'); - return sb.toString(); - } - - // === 下方为用户自定义模块,下次生成会保留 === -} \ No newline at end of file + /** 表字段 : id 字段类型 : bigint(19) */ + private Long id; + + /** 配置模板名称 冗余存储 表字段 : template_name 字段类型 : varchar(200) */ + private String templateName; + + /** uuid 第三方侧记录的模板id 表字段 : template_uuid 字段类型 : varchar(34) */ + private String templateUuid; + + /** id of linkis_ps_configuration_config_key 表字段 : key_id 字段类型 : bigint(19) */ + private Long keyId; + + /** 配置值 表字段 : config_value 字段类型 : varchar(200) */ + private String configValue; + + /** 上限值 表字段 : max_value 字段类型 : varchar(50) */ + private String maxValue; + + /** 下限值（预留） 表字段 : min_value 字段类型 : varchar(50) */ + private String minValue; + + /** 校验正则(预留) 表字段 : validate_range 字段类型 : varchar(50) */ + private String validateRange; + + /** 是否有效 预留 Y/N 表字段 : is_valid 字段类型 : varchar(2) */ + private String isValid; + + /** 创建人 表字段 : create_by 字段类型 : varchar(50) */ + private String createBy; + + /** create time 表字段 : create_time 字段类型 : timestamp(19) 默认值 : CURRENT_TIMESTAMP */ + private Date createTime; + + /** 更新人 表字段 : update_by 字段类型 : varchar(50) */ + private String updateBy; + + /** update time 表字段 : update_time 字段类型 : timestamp(19) 默认值 : CURRENT_TIMESTAMP */ + private Date updateTime; + + /** + * 获取：
+ * 字段：id + * + * @return + */ + public Long getId() { + return id; + } + + /** + * 设置：
+ * 字段：id + * + * @param + */ + public void setId(Long id) { + this.id = id; + } + + /** + * 获取：配置模板名称 冗余存储
+ * 字段：template_name + * + * @return + */ + public String getTemplateName() { + return templateName; + } + + /** + * 设置：配置模板名称 冗余存储
+ * 字段：template_name + * + * @param + */ + public void setTemplateName(String templateName) { + this.templateName = templateName; + } + + /** + * 获取：uuid 第三方侧记录的模板id
+ * 字段：template_uuid + * + * @return + */ + public String getTemplateUuid() { + return templateUuid; + } + + /** + * 设置：uuid 第三方侧记录的模板id
+ * 字段：template_uuid + * + * @param + */ + public void setTemplateUuid(String templateUuid) { + this.templateUuid = templateUuid; + } + + /** + * 获取：id of linkis_ps_configuration_config_key
+ * 字段：key_id + * + * @return + */ + public Long getKeyId() { + return keyId; + } + + /** + * 设置：id of linkis_ps_configuration_config_key
+ * 字段：key_id + * + * @param + */ + public void setKeyId(Long keyId) { + this.keyId = keyId; + } + + /** + * 获取：配置值
+ * 字段：config_value + * + * @return + */ + public String getConfigValue() { + return configValue; + } + + /** + * 设置：配置值
+ * 字段：config_value + * + * @param + */ + public void setConfigValue(String configValue) { + this.configValue = configValue; + } + + /** + * 获取：上限值
+ * 字段：max_value + * + * @return + */ + public String getMaxValue() { + return maxValue; + } + + /** + * 设置：上限值
+ * 字段：max_value + * + * @param + */ + public void setMaxValue(String maxValue) { + this.maxValue = maxValue; + } + + /** + * 获取：下限值（预留）
+ * 字段：min_value + * + * @return + */ + public String getMinValue() { + return minValue; + } + + /** + * 设置：下限值（预留）
+ * 字段：min_value + * + * @param + */ + public void setMinValue(String minValue) { + this.minValue = minValue; + } + + /** + * 获取：校验正则(预留)
+ * 字段：validate_range + * + * @return + */ + public String getValidateRange() { + return validateRange; + } + + /** + * 设置：校验正则(预留)
+ * 字段：validate_range + * + * @param + */ + public void setValidateRange(String validateRange) { + this.validateRange = validateRange; + } + + /** + * 获取：是否有效 预留 Y/N
+ * 字段：is_valid + * + * @return + */ + public String getIsValid() { + return isValid; + } + + /** + * 设置：是否有效 预留 Y/N
+ * 字段：is_valid + * + * @param + */ + public void setIsValid(String isValid) { + this.isValid = isValid; + } + + /** + * 获取：创建人
+ * 字段：create_by + * + * @return + */ + public String getCreateBy() { + return createBy; + } + + /** + * 设置：创建人
+ * 字段：create_by + * + * @param + */ + public void setCreateBy(String createBy) { + this.createBy = createBy; + } + + /** + * 获取：create time
+ * 字段：create_time + * + * @return + */ + public Date getCreateTime() { + return createTime; + } + + /** + * 设置：create time
+ * 字段：create_time + * + * @param + */ + public void setCreateTime(Date createTime) { + this.createTime = createTime; + } + + /** + * 获取：更新人
+ * 字段：update_by + * + * @return + */ + public String getUpdateBy() { + return updateBy; + } + + /** + * 设置：更新人
+ * 字段：update_by + * + * @param + */ + public void setUpdateBy(String updateBy) { + this.updateBy = updateBy; + } + + /** + * 获取：update time
+ * 字段：update_time + * + * @return + */ + public Date getUpdateTime() { + return updateTime; + } + + /** + * 设置：update time
+ * 字段：update_time + * + * @param + */ + public void setUpdateTime(Date updateTime) { + this.updateTime = updateTime; + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + sb.append(getClass().getSimpleName()); + sb.append(" ["); + sb.append("Hash = ").append(hashCode()); + sb.append(", id=").append(id); + sb.append(", templateName=").append(templateName); + sb.append(", templateUuid=").append(templateUuid); + sb.append(", keyId=").append(keyId); + sb.append(", configValue=").append(configValue); + sb.append(", maxValue=").append(maxValue); + sb.append(", minValue=").append(minValue); + sb.append(", validateRange=").append(validateRange); + sb.append(", isValid=").append(isValid); + sb.append(", createBy=").append(createBy); + sb.append(", createTime=").append(createTime); + sb.append(", updateBy=").append(updateBy); + sb.append(", updateTime=").append(updateTime); + sb.append(']'); + return sb.toString(); + } + + // === 下方为用户自定义模块,下次生成会保留 === +} diff --git a/linkis-public-enhancements/linkis-configuration/src/main/java/org/apache/linkis/configuration/entity/TemplateConfigKeyVo.java b/linkis-public-enhancements/linkis-configuration/src/main/java/org/apache/linkis/configuration/entity/TemplateConfigKeyVo.java index 911a2b0b38..915c710404 100644 --- a/linkis-public-enhancements/linkis-configuration/src/main/java/org/apache/linkis/configuration/entity/TemplateConfigKeyVo.java +++ b/linkis-public-enhancements/linkis-configuration/src/main/java/org/apache/linkis/configuration/entity/TemplateConfigKeyVo.java @@ -1,61 +1,64 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.apache.linkis.configuration.entity; +public class TemplateConfigKeyVo { + /** 表字段 : id 字段类型 : bigint(19) */ + private Long id; -public class TemplateConfigKeyVo { + private String key; + + /** 配置值 表字段 : config_value 字段类型 : varchar(200) */ + private String configValue; + + /** 上限值 表字段 : max_value 字段类型 : varchar(50) */ + private String maxValue; + + public Long getId() { + return id; + } + + public void setId(Long id) { + this.id = id; + } + + public String getConfigValue() { + return configValue; + } + + public void setConfigValue(String configValue) { + this.configValue = configValue; + } + + public String getMaxValue() { + return maxValue; + } + + public void setMaxValue(String maxValue) { + this.maxValue = maxValue; + } + + public String getKey() { + return key; + } - /** - * 表字段 : id - * 字段类型 : bigint(19) - */ - private Long id; - - private String key; - - /** - * 配置值 - * 表字段 : config_value - * 字段类型 : varchar(200) - */ - private String configValue; - - /** - * 上限值 - * 表字段 : max_value - * 字段类型 : varchar(50) - */ - private String maxValue; - - public Long getId() { - return id; - } - - public void setId(Long id) { - this.id = id; - } - - - public String getConfigValue() { - return configValue; - } - - public void setConfigValue(String configValue) { - this.configValue = configValue; - } - - public String getMaxValue() { - return maxValue; - } - - public void setMaxValue(String maxValue) { - this.maxValue = maxValue; - } - - public String getKey() { - return key; - } - - public void setKey(String key) { - this.key = key; - } -} \ No newline at end of file + public void setKey(String key) { + this.key = key; + } +} diff --git a/linkis-public-enhancements/linkis-configuration/src/main/java/org/apache/linkis/configuration/restful/api/ConfigurationRestfulApi.java b/linkis-public-enhancements/linkis-configuration/src/main/java/org/apache/linkis/configuration/restful/api/ConfigurationRestfulApi.java index eb1e314307..8824e8ac6b 100644 --- a/linkis-public-enhancements/linkis-configuration/src/main/java/org/apache/linkis/configuration/restful/api/ConfigurationRestfulApi.java +++ b/linkis-public-enhancements/linkis-configuration/src/main/java/org/apache/linkis/configuration/restful/api/ConfigurationRestfulApi.java @@ -155,15 +155,20 @@ public Message getCategory(HttpServletRequest req) { return Message.ok().data("Category", categoryLabelList); } - @ApiOperation(value = "getItemList", notes = "get configuration list by engineType", response = Message.class) + @ApiOperation( + value = "getItemList", + notes = "get configuration list by engineType", + response = Message.class) @RequestMapping(path = "/getItemList", method = RequestMethod.GET) - public Message getItemList(HttpServletRequest req, @RequestParam(value = "engineType") String engineType) throws ConfigurationException { - String userName = ModuleUserUtils.getOperationUser(req, "getItemList with engineType:" + engineType); - List result= configKeyService.getConfigKeyList(engineType); + public Message getItemList( + HttpServletRequest req, @RequestParam(value = "engineType") String engineType) + throws ConfigurationException { + String userName = + ModuleUserUtils.getOperationUser(req, "getItemList with engineType:" + engineType); + List result = configKeyService.getConfigKeyList(engineType); return Message.ok().data("list", result); } - @ApiOperation( value = "createFirstCategory", notes = "create first category", diff --git a/linkis-public-enhancements/linkis-configuration/src/main/java/org/apache/linkis/configuration/restful/api/ConfigurationTemplateRestfulApi.java b/linkis-public-enhancements/linkis-configuration/src/main/java/org/apache/linkis/configuration/restful/api/ConfigurationTemplateRestfulApi.java index 5116609ee1..1f0593d54f 100644 --- a/linkis-public-enhancements/linkis-configuration/src/main/java/org/apache/linkis/configuration/restful/api/ConfigurationTemplateRestfulApi.java +++ b/linkis-public-enhancements/linkis-configuration/src/main/java/org/apache/linkis/configuration/restful/api/ConfigurationTemplateRestfulApi.java @@ -17,71 +17,71 @@ package org.apache.linkis.configuration.restful.api; -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.core.type.TypeReference; -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.github.xiaoymin.knife4j.annotations.ApiOperationSupport; -import io.swagger.annotations.Api; -import io.swagger.annotations.ApiImplicitParam; -import io.swagger.annotations.ApiImplicitParams; -import io.swagger.annotations.ApiOperation; -import org.apache.commons.lang3.StringUtils; import org.apache.linkis.common.conf.Configuration; import org.apache.linkis.common.utils.JsonUtils; -import org.apache.linkis.configuration.entity.*; +import org.apache.linkis.configuration.entity.TemplateConfigKeyVo; import org.apache.linkis.configuration.exception.ConfigurationException; -import org.apache.linkis.configuration.service.CategoryService; -import org.apache.linkis.configuration.service.ConfigKeyService; -import org.apache.linkis.configuration.service.ConfigurationService; -import org.apache.linkis.configuration.util.ConfigurationConfiguration; -import org.apache.linkis.configuration.util.JsonNodeUtil; -import org.apache.linkis.configuration.util.LabelEntityParser; -import org.apache.linkis.manager.label.entity.engine.EngineTypeLabel; -import org.apache.linkis.manager.label.entity.engine.UserCreatorLabel; -import org.apache.linkis.manager.label.utils.LabelUtils; -import org.apache.linkis.server.BDPJettyServerHelper; +import org.apache.linkis.configuration.service.TemplateConfigKeyService; import org.apache.linkis.server.Message; import org.apache.linkis.server.utils.ModuleUserUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; + +import org.apache.commons.lang3.StringUtils; + import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.web.bind.annotation.*; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestMethod; +import org.springframework.web.bind.annotation.RestController; import javax.servlet.http.HttpServletRequest; -import java.io.IOException; -import java.text.MessageFormat; + import java.util.ArrayList; -import java.util.HashMap; import java.util.List; import java.util.Map; -import static org.apache.linkis.configuration.errorcode.LinkisConfigurationErrorCodeSummary.*; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.JsonNode; +import io.swagger.annotations.Api; +import io.swagger.annotations.ApiImplicitParam; +import io.swagger.annotations.ApiImplicitParams; +import io.swagger.annotations.ApiOperation; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; @Api(tags = "configuration template") @RestController @RequestMapping(path = "/configuration/template") public class ConfigurationTemplateRestfulApi { - private static final Logger logger = LoggerFactory.getLogger(ConfigurationTemplateRestfulApi.class); - - @Autowired private ConfigurationService configurationService; + private static final Logger logger = + LoggerFactory.getLogger(ConfigurationTemplateRestfulApi.class); - @Autowired private CategoryService categoryService; + @Autowired private TemplateConfigKeyService templateConfigKeyService; - @Autowired private ConfigKeyService configKeyService; - - @ApiOperation(value = "updateKeyMapping", notes = "query engineconn info list", response = Message.class) + @ApiOperation( + value = "updateKeyMapping", + notes = "query engineconn info list", + response = Message.class) @ApiImplicitParams({ - @ApiImplicitParam(name = "templateUid", dataType = "String", required = true, value = "templateUid"), - @ApiImplicitParam(name = "templateName", dataType = "String", required = true, value = "engine type"), - @ApiImplicitParam(name = "engineType", dataType = "String", required = true,value = "String"), - @ApiImplicitParam(name = "operator", dataType = "String", value = "operator"), - @ApiImplicitParam(name = "isFullMode", dataType = "Boolbean", value = "isFullMode"), - @ApiImplicitParam(name = "itemList", dataType = "Array", value = "itemList"), + @ApiImplicitParam( + name = "templateUid", + dataType = "String", + required = true, + value = "templateUid"), + @ApiImplicitParam( + name = "templateName", + dataType = "String", + required = true, + value = "engine type"), + @ApiImplicitParam(name = "engineType", dataType = "String", required = true, value = "String"), + @ApiImplicitParam(name = "operator", dataType = "String", value = "operator"), + @ApiImplicitParam(name = "isFullMode", dataType = "Boolbean", value = "isFullMode"), + @ApiImplicitParam(name = "itemList", dataType = "Array", value = "itemList"), }) @RequestMapping(path = "/updateKeyMapping", method = RequestMethod.POST) - public Message updateKeyMapping(HttpServletRequest req, @RequestBody JsonNode jsonNode) { + public Message updateKeyMapping(HttpServletRequest req, @RequestBody JsonNode jsonNode) + throws ConfigurationException { String username = ModuleUserUtils.getOperationUser(req, "updateKeyMapping"); String token = ModuleUserUtils.getToken(req); // check special admin token @@ -112,29 +112,163 @@ public Message updateKeyMapping(HttpServletRequest req, @RequestBody JsonNode js if (StringUtils.isBlank(operator)) { return Message.error("parameters:operator can not be empty(请求参数【operator】不能为空)"); } - boolean isFullMode =true; + boolean isFullMode = true; try { - isFullMode=jsonNode.get("isFullMode").asBoolean(); - logger.info("will update by param isFullMode:"+isFullMode); - } - catch (Exception e) - { - logger.info("will update by default isFullMode:"+isFullMode); + isFullMode = jsonNode.get("isFullMode").asBoolean(); + logger.info("will update by param isFullMode:" + isFullMode); + } catch (Exception e) { + logger.info("will update by default isFullMode:" + isFullMode); } - JsonNode itemParms = jsonNode.get("itemList"); List confKeyList = new ArrayList<>(); if (itemParms != null && !itemParms.isNull()) { try { confKeyList = - JsonUtils.jackson() - .readValue(itemParms.toString(), new TypeReference>() {}); + JsonUtils.jackson() + .readValue(itemParms.toString(), new TypeReference>() {}); } catch (JsonProcessingException e) { return Message.error("parameters:itemList parsing failed(请求参数【itemList】解析失败)"); } + } else { + return Message.error("parameters:itemList can not be empty(请求参数【itemList】不能为空)"); + } + + logger.info( + "request parameters templateUid:{}, templateName:{}, engineType:{}, operator:{},isFullMode:{}, itemList:[{}]", + templateUid, + templateName, + engineType, + operator, + itemParms.asText()); + + templateConfigKeyService.updateKeyMapping( + templateUid, templateName, engineType, operator, confKeyList, isFullMode); + return Message.ok(); + } + + @ApiOperation(value = "queryKeyInfoList", notes = "query key info list", response = Message.class) + @ApiImplicitParams({ + @ApiImplicitParam(name = "templateUidList", dataType = "Array", value = "templateUidList"), + }) + @RequestMapping(path = "/queryKeyInfoList", method = RequestMethod.POST) + public Message queryKeyInfoList(HttpServletRequest req, @RequestBody JsonNode jsonNode) + throws ConfigurationException { + String username = ModuleUserUtils.getOperationUser(req, "queryKeyInfoList"); + String token = ModuleUserUtils.getToken(req); + // check special admin token + if (StringUtils.isNotBlank(token)) { + if (!Configuration.isAdminToken(token)) { + logger.warn("Token:{} has no permission to queryKeyInfoList.", token); + return Message.error("Token:" + token + " has no permission to queryKeyInfoList."); + } + } else if (!Configuration.isAdmin(username)) { + logger.warn("User:{} has no permission to queryKeyInfoList.", username); + return Message.error("User:" + username + " has no permission to queryKeyInfoList."); } + JsonNode templateUidListParms = jsonNode.get("templateUidList"); + + List uuidList = new ArrayList<>(); + if (templateUidListParms != null && !templateUidListParms.isNull()) { + try { + uuidList = + JsonUtils.jackson() + .readValue(templateUidListParms.toString(), new TypeReference>() {}); + } catch (JsonProcessingException e) { + return Message.error( + "parameters:templateUidList parsing failed(请求参数【templateUidList】解析失败)"); + } + } else { + return Message.error( + "parameters:templateUidList can not be empty(请求参数【templateUidList】不能为空)"); + } + + List result = templateConfigKeyService.queryKeyInfoList(uuidList); + + return Message.ok().data("list", result); + } + + @ApiOperation(value = "apply", notes = "apply conf template rule", response = Message.class) + @ApiImplicitParams({ + @ApiImplicitParam( + name = "templateUid", + dataType = "String", + required = true, + value = "templateUid"), + @ApiImplicitParam(name = "application", dataType = "String", value = "application"), + @ApiImplicitParam(name = "engineType", dataType = "String", value = "engineType"), + @ApiImplicitParam(name = "engineVersion", dataType = "String", value = "engineVersion"), + @ApiImplicitParam(name = "operator", dataType = "String", value = "operator"), + @ApiImplicitParam(name = "userList", dataType = "Array", value = "userList"), + }) + @RequestMapping(path = "/apply", method = RequestMethod.POST) + public Message apply(HttpServletRequest req, @RequestBody JsonNode jsonNode) + throws ConfigurationException { + String username = ModuleUserUtils.getOperationUser(req, "apply"); + String token = ModuleUserUtils.getToken(req); + // check special admin token + if (StringUtils.isNotBlank(token)) { + if (!Configuration.isAdminToken(token)) { + logger.warn("Token:{} has no permission to apply.", token); + return Message.error("Token:" + token + " has no permission to apply."); + } + } else if (!Configuration.isAdmin(username)) { + logger.warn("User:{} has no permission to apply.", username); + return Message.error("User:" + username + " has no permission to apply."); + } + + String templateUid = jsonNode.get("templateUid").asText(); + String application = jsonNode.get("application").asText(); + String engineType = jsonNode.get("engineType").asText(); + String engineVersion = jsonNode.get("engineVersion").asText(); + String operator = jsonNode.get("operator").asText(); + + if (StringUtils.isBlank(templateUid)) { + return Message.error("parameters:templateUid can not be empty(请求参数【templateUid】不能为空)"); + } + if (StringUtils.isBlank(application)) { + return Message.error("parameters:application can not be empty(请求参数【application】不能为空)"); + } + if (StringUtils.isBlank(engineType)) { + return Message.error("parameters:engineType can not be empty(请求参数【engineType】不能为空)"); + } + if (StringUtils.isBlank(engineVersion)) { + return Message.error("parameters:engineVersion can not be empty(请求参数【engineVersion】不能为空)"); + } + if (StringUtils.isBlank(operator)) { + return Message.error("parameters:operator can not be empty(请求参数【operator】不能为空)"); + } + + JsonNode userParms = jsonNode.get("userList"); + List userList = new ArrayList<>(); + if (userParms != null && !userParms.isNull()) { + try { + userList = + JsonUtils.jackson() + .readValue(userParms.toString(), new TypeReference>() {}); + } catch (JsonProcessingException e) { + return Message.error("parameters:userList parsing failed(请求参数【userList】解析失败)"); + } + } else { + return Message.error("parameters:userList can not be empty(请求参数【userList】不能为空)"); + } + + logger.info( + "request parameters templateUid:{}, application:{}, engineType:{}, engineVersion:{}, operator:{},userList:[{}]", + templateUid, + application, + engineType, + engineVersion, + operator, + String.join(",", userList)); + + Map result = + templateConfigKeyService.apply( + templateUid, application, engineType, engineVersion, operator, userList); + Message message = Message.ok(); + message.getData().putAll(result); + return message; } } diff --git a/linkis-public-enhancements/linkis-configuration/src/main/java/org/apache/linkis/configuration/service/ConfigKeyService.java b/linkis-public-enhancements/linkis-configuration/src/main/java/org/apache/linkis/configuration/service/ConfigKeyService.java index c5e038a16d..0ae650f91e 100644 --- a/linkis-public-enhancements/linkis-configuration/src/main/java/org/apache/linkis/configuration/service/ConfigKeyService.java +++ b/linkis-public-enhancements/linkis-configuration/src/main/java/org/apache/linkis/configuration/service/ConfigKeyService.java @@ -33,9 +33,7 @@ ConfigValue saveConfigValue(ConfigKeyValue configKeyValue, List> labelL List getConfigValue(String configKey, List> labelList) throws ConfigurationException; - - List getConfigKeyList(String engineType) - throws ConfigurationException; + List getConfigKeyList(String engineType) throws ConfigurationException; List deleteConfigValue(String configKey, List> labelList) throws ConfigurationException; diff --git a/linkis-public-enhancements/linkis-configuration/src/main/java/org/apache/linkis/configuration/service/TemplateConfigKeyService.java b/linkis-public-enhancements/linkis-configuration/src/main/java/org/apache/linkis/configuration/service/TemplateConfigKeyService.java index 3e76601d91..1bd60a7896 100644 --- a/linkis-public-enhancements/linkis-configuration/src/main/java/org/apache/linkis/configuration/service/TemplateConfigKeyService.java +++ b/linkis-public-enhancements/linkis-configuration/src/main/java/org/apache/linkis/configuration/service/TemplateConfigKeyService.java @@ -21,11 +21,27 @@ import org.apache.linkis.configuration.exception.ConfigurationException; import java.util.List; +import java.util.Map; public interface TemplateConfigKeyService { + Boolean updateKeyMapping( + String templateUid, + String templateName, + String engineType, + String operator, + List itemList, + Boolean isFullMode) + throws ConfigurationException; - Boolean updateKeyMapping(String templateUid, String templateName, String engineType, - String operator, Boolean isFullMod, List itemList) - throws ConfigurationException; + List queryKeyInfoList(List uuidList) throws ConfigurationException; + + Map apply( + String templateUid, + String application, + String engineType, + String engineVersion, + String operator, + List userList) + throws ConfigurationException; } diff --git a/linkis-public-enhancements/linkis-configuration/src/main/java/org/apache/linkis/configuration/service/impl/TemplateConfigKeyServiceImpl.java b/linkis-public-enhancements/linkis-configuration/src/main/java/org/apache/linkis/configuration/service/impl/TemplateConfigKeyServiceImpl.java index 72b9ef60d7..eb7d6a878f 100644 --- a/linkis-public-enhancements/linkis-configuration/src/main/java/org/apache/linkis/configuration/service/impl/TemplateConfigKeyServiceImpl.java +++ b/linkis-public-enhancements/linkis-configuration/src/main/java/org/apache/linkis/configuration/service/impl/TemplateConfigKeyServiceImpl.java @@ -17,26 +17,27 @@ package org.apache.linkis.configuration.service.impl; +import org.apache.linkis.configuration.dao.ConfigKeyLimitForUserMapper; import org.apache.linkis.configuration.dao.ConfigMapper; import org.apache.linkis.configuration.dao.LabelMapper; import org.apache.linkis.configuration.dao.TemplateConfigKeyMapper; -import org.apache.linkis.configuration.entity.ConfigKey; -import org.apache.linkis.configuration.entity.ConfigKeyValue; -import org.apache.linkis.configuration.entity.TemplateConfigKey; -import org.apache.linkis.configuration.entity.TemplateConfigKeyVo; +import org.apache.linkis.configuration.entity.*; import org.apache.linkis.configuration.exception.ConfigurationException; import org.apache.linkis.configuration.service.ConfigurationService; import org.apache.linkis.configuration.service.TemplateConfigKeyService; +import org.apache.linkis.configuration.util.LabelEntityParser; import org.apache.linkis.configuration.validate.ValidatorManager; -import org.apache.linkis.manager.common.entity.persistence.PersistencerEcNodeInfo; import org.apache.linkis.manager.label.builder.CombinedLabelBuilder; +import org.apache.linkis.manager.label.entity.CombinedLabel; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; import java.text.MessageFormat; import java.util.ArrayList; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.stream.Collectors; @@ -55,85 +56,289 @@ public class TemplateConfigKeyServiceImpl implements TemplateConfigKeyService { @Autowired private TemplateConfigKeyMapper templateConfigKeyMapper; - @Autowired private ConfigurationService configurationService; @Autowired private ValidatorManager validatorManager; + @Autowired + private ConfigKeyLimitForUserMapper configKeyLimitForUserMapper; private CombinedLabelBuilder combinedLabelBuilder = new CombinedLabelBuilder(); + @Override + @Transactional + public Boolean updateKeyMapping( + String templateUid, + String templateName, + String engineType, + String operator, + List itemList, + Boolean isFullMode) + throws ConfigurationException { + // isFullMode true + // 查询对应的数据 并做数据合法性检查 + List keyList = itemList.stream().map(e -> e.getKey()).collect(Collectors.toList()); + List configKeyList = + configMapper.selectKeyByEngineTypeAndKeyList(engineType, keyList); + // 待更新的key id 列表 + List keyIdList = configKeyList.stream().map(e -> e.getId()).collect(Collectors.toList()); + if (configKeyList.size() != itemList.size()) { + String msg = + MessageFormat.format( + "The num of config item data from the DB is inconsistent(DB中获取到的配置数据条数不一致) :" + + "engineType:{0}, input keyList size:{1}, db keyList size:{2}", + engineType, keyList.size(), configKeyList.size()); + throw new ConfigurationException(msg); + } + // 组装更新 + List toUpdateOrInsertList = new ArrayList<>(); + + // map k:v---> key：ConfigKey + Map configKeyMap = + configKeyList.stream().collect(Collectors.toMap(ConfigKey::getKey, item -> item)); + for (TemplateConfigKeyVo item : itemList) { + + String key = item.getKey(); + ConfigKey temp = configKeyMap.get(item.getKey()); + Long keyId = temp.getId(); + String validateType = temp.getValidateType(); + String validateRange = temp.getValidateRange(); + String configValue = item.getConfigValue(); + String maxValue = item.getMaxValue(); - @Override - public Boolean updateKeyMapping(String templateUid, String templateName, String engineType, - String operator, Boolean isFullMode, List itemList) throws ConfigurationException { - //isFullMode true - //查询对应的数据 并做数据合法性检查 - List keyList=itemList.stream().map(e->e.getKey()).collect(Collectors.toList()); - List configKeyList=configMapper.selectKeyByEngineTypeAndKeyList(engineType,keyList); - //待更新的key id 列表 - List keyIdList=configKeyList.stream().map(e->e.getId()).collect(Collectors.toList()); - if(configKeyList.size()!=itemList.size()) - { + if (!validatorManager + .getOrCreateValidator(validateType) + .validate(configValue, validateRange)) { + String msg = + MessageFormat.format( + "Parameter configValue verification failed(参数configValue校验失败):" + + "key:{0}, ValidateType:{1}, ValidateRange:{},ConfigValue:{}", + key, validateType, validateRange, configValue); + throw new ConfigurationException(msg); + } + if (!validatorManager.getOrCreateValidator(validateType).validate(maxValue, validateRange)) { + String msg = + MessageFormat.format( + "Parameter maxValue verification failed(参数maxValue校验失败):" + + "key:{0}, ValidateType:{1}, ValidateRange:{},ConfigValue:{}", + key, validateType, validateRange, maxValue); + throw new ConfigurationException(msg); + } + TemplateConfigKey templateConfigKey = new TemplateConfigKey(); + templateConfigKey.setTemplateName(templateName); + templateConfigKey.setTemplateUuid(templateUid); + templateConfigKey.setKeyId(keyId); + templateConfigKey.setConfigValue(configValue); + templateConfigKey.setMaxValue(maxValue); + templateConfigKey.setCreateBy(operator); + templateConfigKey.setUpdateBy(operator); + toUpdateOrInsertList.add(templateConfigKey); + } + // 根据不同模式更新数据 + if (isFullMode) { + // 之前在数据库中的数据 需要移除的 + List oldList = + templateConfigKeyMapper.selectListByTemplateUuid(templateUid); + List needToRemoveList = + oldList.stream() + .filter( + item -> { + return !keyIdList.contains(item.getKeyId()); + }) + .map(e -> e.getKeyId()) + .collect(Collectors.toList()); + logger.info( + "Try to remove old data:[" + needToRemoveList + "] for templateUid:" + templateUid); + templateConfigKeyMapper.deleteByTemplateUuidAndKeyIdList(templateUid, needToRemoveList); + } + + templateConfigKeyMapper.batchInsertOrUpdateList(toUpdateOrInsertList); + + return true; } - //组装更新 - List toUpdateOrInsertList=new ArrayList<>(); - - // map k:v---> key：ConfigKey - Map configKeyMap = - configKeyList.stream() - .collect(Collectors.toMap(ConfigKey::getKey, item -> item)); - for (TemplateConfigKeyVo item : itemList) { - - String key=item.getKey(); - ConfigKey temp = configKeyMap.get(item.getKey()); - String validateType=temp.getValidateType(); - String validateRange=temp.getValidateRange(); - String configValue=item.getConfigValue(); - String maxValue=item.getMaxValue(); - - if (!validatorManager - .getOrCreateValidator(validateType) - .validate(configValue, validateRange) - ) { - String msg= MessageFormat.format("Parameter configValue verification failed(参数configValue校验失败):" + - "key:{0}, ValidateType:{1}, ValidateRange:{},ConfigValue:{}", - key,validateType,validateRange,configValue); - throw new ConfigurationException(msg); - } - if (!validatorManager - .getOrCreateValidator(validateType) - .validate(maxValue, validateRange) - ) { - String msg= MessageFormat.format("Parameter maxValue verification failed(参数maxValue校验失败):" + - "key:{0}, ValidateType:{1}, ValidateRange:{},ConfigValue:{}", - key,validateType,validateRange,maxValue); - throw new ConfigurationException(msg); - } - - TemplateConfigKey templateConfigKey=new TemplateConfigKey(); - - - } - //根据不同模式更新数据 - if(isFullMode) - { - //之前在数据库中的数据 需要移除的 - List oldList =templateConfigKeyMapper.selectListByTemplateUuid(templateUid); - List needToRemoveList=oldList.stream().filter(item ->{ - return !keyIdList.contains(item.getKeyId()); - }).map(e->e.getKeyId()).collect(Collectors.toList()); - logger.info("Try to remove old data:["+needToRemoveList+"] for templateUid:"+templateUid); - templateConfigKeyMapper.deleteByTemplateUuidAndKeyIdList(templateUid,needToRemoveList); - } - - - itemList.forEach(); - templateConfigKeyMapper.batchInsertList(templateUid,needToRemoveList); - - } + @Override + public List queryKeyInfoList(List uuidList) throws ConfigurationException { + List result = new ArrayList<>(); + + List templateConfigKeyList = + templateConfigKeyMapper.selectListByTemplateUuidList(uuidList); + + Map> templateConfigKeyListGroupByUuid = + templateConfigKeyList.stream() + .collect(Collectors.groupingBy(TemplateConfigKey::getTemplateUuid)); + + List keyIdList = + templateConfigKeyList.stream().map(e -> e.getId()).distinct().collect(Collectors.toList()); + List configKeyList = configMapper.selectKeyByKeyIdList(keyIdList); + // map k:v---> keyId：ConfigKey + Map configKeyMap = + configKeyList.stream().collect(Collectors.toMap(ConfigKey::getId, item -> item)); + + for (String uuid : templateConfigKeyListGroupByUuid.keySet()) { + Map item = new HashMap(); + List keys = new ArrayList<>(); + item.put("uuid", uuid); + + List group = templateConfigKeyListGroupByUuid.get(uuid); + for (TemplateConfigKey templateConfigKey : group) { + // Map temp = BDPJettyServerHelper.jacksonJson().convertValue(templateConfigKey, Map.class); + Map temp = new HashMap(); + temp.put("configValue", templateConfigKey.getConfigValue()); + temp.put("maxValue", templateConfigKey.getMaxValue()); + temp.put("createBy", templateConfigKey.getCreateBy()); + temp.put("createTime", templateConfigKey.getCreateTime()); + temp.put("updateBy", templateConfigKey.getUpdateBy()); + temp.put("updateTime", templateConfigKey.getUpdateTime()); + temp.put("keyId", templateConfigKey.getKeyId()); + ConfigKey info = configKeyMap.get(templateConfigKey.getKeyId()); + temp.put("key", info.getKey()); + keys.add(temp); + } + item.put("keyList", keys); + result.add(item); + } + return result; + } + + // @Override + // public List apply1(String templateUid, String application, String engineType, + // String engineVersion, String operator, List userList) + // throws ConfigurationException { + // // 确认用户该标签是否都存在 + // // combined_label_value 组合标签 combined_userCreator_engineType 如 + // hadoop-IDE,spark-2.4.3 + // String combinedKey = LabelEntityParser.COMBINED_USERCREATOR_ENGINETYPE; + // String valueFormat = "{}-{},{}-{}"; + // + // List configLabelList = userList.stream().filter(e -> + // StringUtils.isNotBlank(e)) + // .map(user -> { + //// ConfigLabel label = new ConfigLabel(); + //// label.setLabelKey(combinedKey); + //// String + // value=MessageFormat.format(valueFormat,user,application,engineType,engineVersion); + //// label.setStringValue(value); + //// label.setFeature(Feature.OPTIONAL); + //// label.setLabelValueSize(); + // CombinedLabel combinedLabel = + // configurationService.generateCombinedLabel(engineType, + // engineVersion, user, application); + // ConfigLabel configLabel = + // LabelEntityParser.parseToConfigLabel(combinedLabel); + // return configLabel; + // }) + // .collect(Collectors.toList()); + // // 尝试批量创建用户label信息 + // labelMapper.batchInsertLabel(configLabelList); + // List valueList = configLabelList.stream().map(e -> + // e.getStringValue()).collect(Collectors.toList()); + // List labelList = + // labelMapper.selectUserCreatorEngineTypeLabelList(valueList); + // + // //组装批量更新的configvalue + // List needUpdataList = new ArrayList<>(); + // + // ConfigValue configValue = null; + // configValue = new ConfigValue(); + // configValue.setConfigKeyId(configKey.getId()); + // configValue.setConfigValue(configKeyValue.getConfigValue()); + // configValue.setConfigLabelId(configLabel.getId()); + // //获取label id + // + // //更新对应的配置值 + // + // } + + @Override + public Map apply( + String templateUid, + String application, + String engineType, + String engineVersion, + String operator, + List userList) + throws ConfigurationException { + List successList = new ArrayList<>(); + List errorList = new ArrayList<>(); + + // get the associated config itsm list + List templateUuidList = new ArrayList<>(); + templateUuidList.add(templateUid); + List templateConfigKeyList = + templateConfigKeyMapper.selectListByTemplateUuidList(templateUuidList); + + for (String user : userList) { + // try to create combined_userCreator_engineType label for user + Map res = new HashMap(); + res.put("user", user); + try { + CombinedLabel combinedLabel = + configurationService.generateCombinedLabel( + engineType, engineVersion, user, application); + String conbinedLabelKey = combinedLabel.getLabelKey(); + String conbinedLabelStringValue = combinedLabel.getStringValue(); + ConfigLabel configLabel = + labelMapper.getLabelByKeyValue(conbinedLabelKey, conbinedLabelStringValue); + if (null == configLabel || configLabel.getId() < 0) { + configLabel = LabelEntityParser.parseToConfigLabel(combinedLabel); + labelMapper.insertLabel(configLabel); + logger.info("succeed to create label: {}", configLabel.getStringValue()); + } + + // batch update config value + List configValues = new ArrayList<>(); + + List configKeyLimitForUsers = new ArrayList<>(); + + for (TemplateConfigKey templateConfigKey : templateConfigKeyList) { + Long keyId = templateConfigKey.getKeyId(); + String uuid = templateConfigKey.getTemplateUuid(); + + ConfigValue configValue = new ConfigValue(); + configValue.setConfigKeyId(keyId); + configValue.setConfigValue(templateConfigKey.getConfigValue()); + configValue.setConfigLabelId(configLabel.getId()); + configValues.add(configValue); + + ConfigKeyLimitForUser configKeyLimitForUser = new ConfigKeyLimitForUser(); + configKeyLimitForUser.setUserName(user); + configKeyLimitForUser.setCombinedLabelValue(configLabel.getStringValue()); + configKeyLimitForUser.setKeyId(keyId); + configKeyLimitForUser.setLatestUpdateTemplateUuid(uuid); + configKeyLimitForUser.setCreateBy(operator); + configKeyLimitForUser.setUpdateBy(operator); + configKeyLimitForUsers.add(configKeyLimitForUser); + } + configMapper.batchInsertValue(configValues); + + // batch update user ConfigKeyLimitForUserMapper + configKeyLimitForUserMapper.batchInsertList(configKeyLimitForUsers); + + successList.add(res); + } catch (Exception e) { + logger.warn("try to update configurations for user:" + user + " with error", e); + res.put("msg", e.getMessage()); + errorList.add(res); + } + } + + Map result = new HashMap<>(); + + Map successResult = new HashMap<>(); + Map errorResult = new HashMap<>(); + + successResult.put("num", successList.size()); + successResult.put("infoList", successList); + + errorResult.put("num", errorList.size()); + errorResult.put("infoList", errorList); + + result.put("success", successResult); + result.put("error", errorResult); + return result; + } } diff --git a/linkis-public-enhancements/linkis-configuration/src/main/resources/mapper/common/ConfigKeyLimitForUserMapper.xml b/linkis-public-enhancements/linkis-configuration/src/main/resources/mapper/common/ConfigKeyLimitForUserMapper.xml index 7c63950613..d58a0ca361 100644 --- a/linkis-public-enhancements/linkis-configuration/src/main/resources/mapper/common/ConfigKeyLimitForUserMapper.xml +++ b/linkis-public-enhancements/linkis-configuration/src/main/resources/mapper/common/ConfigKeyLimitForUserMapper.xml @@ -1,4 +1,20 @@ + @@ -170,4 +186,41 @@ + + + insert into config_key_limit_for_user (user_name, combined_label_value, + key_id, latest_update_template_uuid, + create_by, create_time, update_by, + update_time) + values + + ( + #{item.userName,jdbcType=VARCHAR}, #{item.combinedLabelValue,jdbcType=VARCHAR}, + #{item.keyId,jdbcType=BIGINT}, #{item.latestUpdateTemplateUuid,jdbcType=VARCHAR}, + #{item.createBy,jdbcType=VARCHAR}, + + + #{item.createTime}, + + + now(), + + , + #{item.updateBy,jdbcType=VARCHAR}, + + + #{item.updateTime}, + + + now(), + + + ) + + on duplicate key update + latest_update_template_uuid =values(latest_update_template_uuid), + update_by =values(update_by), + update_time= now() + + diff --git a/linkis-public-enhancements/linkis-configuration/src/main/resources/mapper/common/ConfigMapper.xml b/linkis-public-enhancements/linkis-configuration/src/main/resources/mapper/common/ConfigMapper.xml index efec75659a..82d71a2b01 100644 --- a/linkis-public-enhancements/linkis-configuration/src/main/resources/mapper/common/ConfigMapper.xml +++ b/linkis-public-enhancements/linkis-configuration/src/main/resources/mapper/common/ConfigMapper.xml @@ -182,7 +182,15 @@ - + SELECT * FROM linkis_ps_configuration_config_value WHERE id = #{id} diff --git a/linkis-public-enhancements/linkis-configuration/src/main/resources/mapper/common/LabelMapper.xml b/linkis-public-enhancements/linkis-configuration/src/main/resources/mapper/common/LabelMapper.xml index d4cea8e29a..34783107fb 100644 --- a/linkis-public-enhancements/linkis-configuration/src/main/resources/mapper/common/LabelMapper.xml +++ b/linkis-public-enhancements/linkis-configuration/src/main/resources/mapper/common/LabelMapper.xml @@ -50,6 +50,16 @@ where label_key = #{labelKey} and label_value = #{stringValue} + + INSERT INTO linkis_cg_manager_label( label_key, label_value,label_feature, label_value_size, update_time, create_time) @@ -57,6 +67,17 @@ #{labelKey}, #{stringValue}, #{feature}, #{labelValueSize}, now(), now()) + + + INSERT IGNORE INTO linkis_cg_manager_label( + label_key, label_value,label_feature, label_value_size, update_time, create_time) + VALUES + + #{item.labelKey}, #{item.stringValue}, #{item.feature}, #{item.labelValueSize}, now(), now() + + + + DELETE FROM linkis_cg_manager_label WHERE id IN diff --git a/linkis-public-enhancements/linkis-configuration/src/main/resources/mapper/common/TemplateConfigKeyMapper.xml b/linkis-public-enhancements/linkis-configuration/src/main/resources/mapper/common/TemplateConfigKeyMapper.xml index 113e7b4948..165e619bb1 100644 --- a/linkis-public-enhancements/linkis-configuration/src/main/resources/mapper/common/TemplateConfigKeyMapper.xml +++ b/linkis-public-enhancements/linkis-configuration/src/main/resources/mapper/common/TemplateConfigKeyMapper.xml @@ -1,4 +1,21 @@ + + @@ -225,4 +242,54 @@ #{item} + + + insert into template_config_key (template_name, template_uuid, + key_id, config_value, max_value, + create_by, create_time, update_by, update_time + ) + values + + ( + #{item.templateName,jdbcType=VARCHAR}, #{item.templateUuid,jdbcType=VARCHAR}, + #{item.keyId,jdbcType=BIGINT}, #{item.configValue,jdbcType=VARCHAR}, #{item.maxValue,jdbcType=VARCHAR}, + #{item.createBy,jdbcType=VARCHAR}, + + + #{item.createTime}, + + + now(), + + , + #{item.updateBy,jdbcType=VARCHAR}, + + + #{item.updateTime}, + + + now(), + + + ) + + on duplicate key update + template_name =values(template_name), + config_value =values(config_value), + max_value =values(max_value), + update_by=values(update_by), + update_time= now() + + + + From abba3d2863c9eef2ec66008a6dec87440f70869c Mon Sep 17 00:00:00 2001 From: peacewong Date: Wed, 21 Jun 2023 15:20:43 +0800 Subject: [PATCH 171/261] remove try and warn --- .../hive/executor/HiveEngineConnExecutor.scala | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/linkis-engineconn-plugins/hive/src/main/scala/org/apache/linkis/engineplugin/hive/executor/HiveEngineConnExecutor.scala b/linkis-engineconn-plugins/hive/src/main/scala/org/apache/linkis/engineplugin/hive/executor/HiveEngineConnExecutor.scala index d73cd9a6c7..9ce82e7558 100644 --- a/linkis-engineconn-plugins/hive/src/main/scala/org/apache/linkis/engineplugin/hive/executor/HiveEngineConnExecutor.scala +++ b/linkis-engineconn-plugins/hive/src/main/scala/org/apache/linkis/engineplugin/hive/executor/HiveEngineConnExecutor.scala @@ -640,12 +640,10 @@ class HiveDriverProxy(driver: Any) extends Logging { } def getResults(res: util.List[_]): Boolean = { - Utils.tryAndWarn { - driver.getClass - .getMethod("getResults", classOf[util.List[_]]) - .invoke(driver, res.asInstanceOf[AnyRef]) - .asInstanceOf[Boolean] - } + driver.getClass + .getMethod("getResults", classOf[util.List[_]]) + .invoke(driver, res.asInstanceOf[AnyRef]) + .asInstanceOf[Boolean] } def close(): Unit = { From 77eba0116fc4b83aba962fcee9f6e598abb98856 Mon Sep 17 00:00:00 2001 From: casionone Date: Wed, 21 Jun 2023 17:27:36 +0800 Subject: [PATCH 172/261] add test for TemplateConfigKey dao --- .../configuration/entity/ConfigKey.java | 26 ++++++- .../restful/api/ConfigurationRestfulApi.java | 4 +- .../impl/TemplateConfigKeyServiceImpl.java | 13 +++- .../resources/mapper/common/ConfigMapper.xml | 3 +- .../dao/ConfigKeyLimitForUserMapperTest.java | 45 ++++++++++++ .../dao/TemplateConfigKeyMapperTest.java | 73 +++++++++++++++++++ .../src/test/resources/create.sql | 38 ++++++++++ pom.xml | 8 ++ 8 files changed, 205 insertions(+), 5 deletions(-) create mode 100644 linkis-public-enhancements/linkis-configuration/src/test/java/org/apache/linkis/configuration/dao/ConfigKeyLimitForUserMapperTest.java create mode 100644 linkis-public-enhancements/linkis-configuration/src/test/java/org/apache/linkis/configuration/dao/TemplateConfigKeyMapperTest.java diff --git a/linkis-public-enhancements/linkis-configuration/src/main/java/org/apache/linkis/configuration/entity/ConfigKey.java b/linkis-public-enhancements/linkis-configuration/src/main/java/org/apache/linkis/configuration/entity/ConfigKey.java index 8c54e4663c..384259a342 100644 --- a/linkis-public-enhancements/linkis-configuration/src/main/java/org/apache/linkis/configuration/entity/ConfigKey.java +++ b/linkis-public-enhancements/linkis-configuration/src/main/java/org/apache/linkis/configuration/entity/ConfigKey.java @@ -43,7 +43,12 @@ public class ConfigKey { private String treeName; - // 0 none/ 1 with mix /2 with max / 3 min and max both + /* + 0 none + 1 with mix + 2 with max + 3 min and max both + */ private Integer boundaryType; public String getEngineType() { @@ -149,4 +154,23 @@ public Integer getBoundaryType() { public void setBoundaryType(Integer boundaryType) { this.boundaryType = boundaryType; } + + @Override + public String toString() { + return "ConfigKey{" + + "id=" + id + + ", key='" + key + '\'' + + ", description='" + description + '\'' + + ", name='" + name + '\'' + + ", engineType='" + engineType + '\'' + + ", defaultValue='" + defaultValue + '\'' + + ", validateType='" + validateType + '\'' + + ", validateRange='" + validateRange + '\'' + + ", isAdvanced=" + isAdvanced + + ", isHidden=" + isHidden + + ", level=" + level + + ", treeName='" + treeName + '\'' + + ", boundaryType=" + boundaryType + + '}'; + } } diff --git a/linkis-public-enhancements/linkis-configuration/src/main/java/org/apache/linkis/configuration/restful/api/ConfigurationRestfulApi.java b/linkis-public-enhancements/linkis-configuration/src/main/java/org/apache/linkis/configuration/restful/api/ConfigurationRestfulApi.java index 8824e8ac6b..8d4a9a3455 100644 --- a/linkis-public-enhancements/linkis-configuration/src/main/java/org/apache/linkis/configuration/restful/api/ConfigurationRestfulApi.java +++ b/linkis-public-enhancements/linkis-configuration/src/main/java/org/apache/linkis/configuration/restful/api/ConfigurationRestfulApi.java @@ -152,6 +152,7 @@ public Message getFullTreesByAppName( public Message getCategory(HttpServletRequest req) { List categoryLabelList = categoryService.getAllCategory(req.getHeader("Content-Language")); + return Message.ok().data("Category", categoryLabelList); } @@ -166,9 +167,10 @@ public Message getItemList( String userName = ModuleUserUtils.getOperationUser(req, "getItemList with engineType:" + engineType); List result = configKeyService.getConfigKeyList(engineType); - return Message.ok().data("list", result); + return Message.ok().data("itemList", result); } + @ApiOperation( value = "createFirstCategory", notes = "create first category", diff --git a/linkis-public-enhancements/linkis-configuration/src/main/java/org/apache/linkis/configuration/service/impl/TemplateConfigKeyServiceImpl.java b/linkis-public-enhancements/linkis-configuration/src/main/java/org/apache/linkis/configuration/service/impl/TemplateConfigKeyServiceImpl.java index eb7d6a878f..4c65f2d949 100644 --- a/linkis-public-enhancements/linkis-configuration/src/main/java/org/apache/linkis/configuration/service/impl/TemplateConfigKeyServiceImpl.java +++ b/linkis-public-enhancements/linkis-configuration/src/main/java/org/apache/linkis/configuration/service/impl/TemplateConfigKeyServiceImpl.java @@ -181,12 +181,13 @@ public List queryKeyInfoList(List uuidList) throws Configuration for (String uuid : templateConfigKeyListGroupByUuid.keySet()) { Map item = new HashMap(); List keys = new ArrayList<>(); - item.put("uuid", uuid); + item.put("templateUid", uuid); List group = templateConfigKeyListGroupByUuid.get(uuid); for (TemplateConfigKey templateConfigKey : group) { // Map temp = BDPJettyServerHelper.jacksonJson().convertValue(templateConfigKey, Map.class); Map temp = new HashMap(); + temp.put("configValue", templateConfigKey.getConfigValue()); temp.put("maxValue", templateConfigKey.getMaxValue()); temp.put("createBy", templateConfigKey.getCreateBy()); @@ -194,11 +195,19 @@ public List queryKeyInfoList(List uuidList) throws Configuration temp.put("updateBy", templateConfigKey.getUpdateBy()); temp.put("updateTime", templateConfigKey.getUpdateTime()); temp.put("keyId", templateConfigKey.getKeyId()); + ConfigKey info = configKeyMap.get(templateConfigKey.getKeyId()); temp.put("key", info.getKey()); + temp.put("name", info.getName()); + temp.put("description", info.getDescription()); + temp.put("engineType", info.getEngineType()); + temp.put("validateType", info.getValidateType()); + temp.put("validateRange", info.getValidateRange()); + temp.put("boundaryType",info.getBoundaryType()); + keys.add(temp); } - item.put("keyList", keys); + item.put("itemList", keys); result.add(item); } return result; diff --git a/linkis-public-enhancements/linkis-configuration/src/main/resources/mapper/common/ConfigMapper.xml b/linkis-public-enhancements/linkis-configuration/src/main/resources/mapper/common/ConfigMapper.xml index 82d71a2b01..9ce1e99e82 100644 --- a/linkis-public-enhancements/linkis-configuration/src/main/resources/mapper/common/ConfigMapper.xml +++ b/linkis-public-enhancements/linkis-configuration/src/main/resources/mapper/common/ConfigMapper.xml @@ -167,7 +167,8 @@ diff --git a/linkis-public-enhancements/linkis-configuration/src/test/java/org/apache/linkis/configuration/dao/ConfigKeyLimitForUserMapperTest.java b/linkis-public-enhancements/linkis-configuration/src/test/java/org/apache/linkis/configuration/dao/ConfigKeyLimitForUserMapperTest.java new file mode 100644 index 0000000000..de54a2e7c9 --- /dev/null +++ b/linkis-public-enhancements/linkis-configuration/src/test/java/org/apache/linkis/configuration/dao/ConfigKeyLimitForUserMapperTest.java @@ -0,0 +1,45 @@ +package org.apache.linkis.configuration.dao; +import java.util.Date; + +import org.apache.linkis.configuration.entity.ConfigKeyLimitForUser; +import org.apache.linkis.configuration.entity.TemplateConfigKey; +import org.instancio.Instancio; +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; + +import java.util.List; +import java.util.UUID; + +import static org.junit.jupiter.api.Assertions.*; + +class ConfigKeyLimitForUserMapperTest extends BaseDaoTest { + @Autowired + ConfigKeyLimitForUserMapper configKeyLimitForUserMapper; + + String uuid = UUID.randomUUID().toString(); + String name = "for-test"; + + private List initData() { + List list = Instancio.ofList(ConfigKeyLimitForUser.class).size(9).create(); + ConfigKeyLimitForUser configKeyLimitForUser = new ConfigKeyLimitForUser(); + configKeyLimitForUser.setUserName("testuser"); + configKeyLimitForUser.setCombinedLabelValue("IDE-hadoop,spark-2.3.3"); + configKeyLimitForUser.setKeyId(1L); + configKeyLimitForUser.setLatestUpdateTemplateUuid(uuid); + configKeyLimitForUser.setCreateBy("test"); + configKeyLimitForUser.setUpdateBy("test"); + list.add(configKeyLimitForUser); + configKeyLimitForUserMapper.batchInsertList(list); + return list; + } + + + @Test + void batchInsertOrUpdateListTest() { + List list=initData(); + list.get(1).setLatestUpdateTemplateUuid("123456"); + int isOk=configKeyLimitForUserMapper.batchInsertOrUpdateList(list); + assertEquals(isOk, 1); + + } +} \ No newline at end of file diff --git a/linkis-public-enhancements/linkis-configuration/src/test/java/org/apache/linkis/configuration/dao/TemplateConfigKeyMapperTest.java b/linkis-public-enhancements/linkis-configuration/src/test/java/org/apache/linkis/configuration/dao/TemplateConfigKeyMapperTest.java new file mode 100644 index 0000000000..0042a13136 --- /dev/null +++ b/linkis-public-enhancements/linkis-configuration/src/test/java/org/apache/linkis/configuration/dao/TemplateConfigKeyMapperTest.java @@ -0,0 +1,73 @@ +package org.apache.linkis.configuration.dao; + +import org.apache.linkis.configuration.entity.TemplateConfigKey; +import org.instancio.Instancio; +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; + +import java.util.ArrayList; +import java.util.List; +import java.util.UUID; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +class TemplateConfigKeyMapperTest extends BaseDaoTest { + + @Autowired + TemplateConfigKeyMapper templateConfigKeyMapper; + String uuid = UUID.randomUUID().toString(); + String name = "for-test"; + + private List initData() { + List list = Instancio.ofList(TemplateConfigKey.class).size(9).create(); + TemplateConfigKey templateConfigKey = new TemplateConfigKey(); + templateConfigKey.setTemplateName(name); + templateConfigKey.setTemplateUuid(uuid); + templateConfigKey.setKeyId(1L); + templateConfigKey.setConfigValue("3"); + templateConfigKey.setMaxValue("8"); + templateConfigKey.setCreateBy("test"); + templateConfigKey.setUpdateBy("test"); + list.add(templateConfigKey); + templateConfigKeyMapper.batchInsertList(list); + return list; + } + + + @Test + void selectListByTemplateUuid() { + initData(); + List res = templateConfigKeyMapper.selectListByTemplateUuid(uuid); + assertEquals(res.size(), 1); + assertEquals(res.get(0).getTemplateName(), name); + + } + + @Test + void deleteByTemplateUuidAndKeyIdList() { + List list = initData(); + List KeyIdList = new ArrayList<>(); + KeyIdList.add(1L); + int num = templateConfigKeyMapper.deleteByTemplateUuidAndKeyIdList(uuid, KeyIdList); + assertEquals(num, 1); + + } + + @Test + void batchInsertOrUpdateList() { + List list = initData(); + list.get(1).setConfigValue("20"); + int isOK = templateConfigKeyMapper.batchInsertOrUpdateList(list); + assertEquals(isOK, 1); + } + + @Test + void selectListByTemplateUuidList() { + List list = initData(); + List templateUuidList = new ArrayList<>(); + templateUuidList.add(uuid); + templateUuidList.add("123456"); + List res = templateConfigKeyMapper.selectListByTemplateUuidList(templateUuidList); + assertEquals(res.size(), 1); + } +} \ No newline at end of file diff --git a/linkis-public-enhancements/linkis-configuration/src/test/resources/create.sql b/linkis-public-enhancements/linkis-configuration/src/test/resources/create.sql index 75a3ad195a..8a33769d92 100644 --- a/linkis-public-enhancements/linkis-configuration/src/test/resources/create.sql +++ b/linkis-public-enhancements/linkis-configuration/src/test/resources/create.sql @@ -48,6 +48,7 @@ CREATE TABLE `linkis_ps_configuration_config_key` `is_advanced` tinyint(1) DEFAULT NULL COMMENT 'Whether it is an advanced parameter. If set to 1(true), parameters would be displayed only when user choose to do so', `level` tinyint(1) DEFAULT NULL COMMENT 'Basis for displaying sorting in the front-end. Higher the level is, higher the rank the parameter gets', `treeName` varchar(20) DEFAULT NULL COMMENT 'Reserved field, representing the subdirectory of engineType', + `id` int(2) NOT NULL COMMENT '0 none/ 1 with mix /2 with max / 3 min and max both', PRIMARY KEY (`id`) ); @@ -86,4 +87,41 @@ CREATE TABLE `linkis_ps_configuration_category` `create_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP, PRIMARY KEY (`id`), UNIQUE INDEX (`label_id`) +); + + +DROP TABLE IF EXISTS `linkis_ps_configuration_template_config_key`; +CREATE TABLE `linkis_ps_configuration_template_config_key` ( + `id` int(20) NOT NULL AUTO_INCREMENT, + `template_name` varchar(200) NOT NULL COMMENT '配置模板名称 冗余存储', + `template_uuid` varchar(34) NOT NULL COMMENT 'uuid 第三方侧记录的模板id', + `key_id` int(20) NOT NULL COMMENT 'id of linkis_ps_configuration_config_key', + `config_value` varchar(200) NOT NULL COMMENT '配置值', + `max_value` varchar(50) NOT NULL COMMENT '上限值', + `min_value` varchar(50) NULL DEFAULT NULL COMMENT '下限值（预留）', + `validate_range` varchar(50) NULL DEFAULT NULL COMMENT '校验正则(预留) ', + `is_valid` varchar(2) NULL DEFAULT NULL COMMENT '是否有效 预留 Y/N', + `create_by` varchar(50) NOT NULL COMMENT '创建人', + `create_time` datetime DEFAULT CURRENT_TIMESTAMP COMMENT 'create time', + `update_by` varchar(50) NULL DEFAULT NULL COMMENT '更新人', + `update_time` datetime DEFAULT CURRENT_TIMESTAMP COMMENT 'update time', + PRIMARY KEY (`id`), + UNIQUE INDEX `uniq_tid_kid` (`template_uuid`, `key_id`) +); + + +DROP TABLE IF EXISTS `linkis_ps_configuration_key_limit_for_user`; +CREATE TABLE `linkis_ps_configuration_key_limit_for_user` ( + `id` int(20) NOT NULL AUTO_INCREMENT, + `user_name` varchar(50) NOT NULL COMMENT '用户名', + `combined_label_value` varchar(200) NOT NULL COMMENT '组合标签 combined_userCreator_engineType 如 hadoop-IDE,spark-2.4.3', + `key_id` int(20) NOT NULL COMMENT 'id of linkis_ps_configuration_config_key', + `latest_update_template_uuid` varchar(34) NOT NULL COMMENT 'uuid 第三方侧记录的模板id', + `is_valid` varchar(2) NULL DEFAULT NULL COMMENT '是否有效 预留 Y/N', + `create_by` varchar(50) NOT NULL COMMENT '创建人', + `create_time` datetime DEFAULT CURRENT_TIMESTAMP COMMENT 'create time', + `update_by` varchar(50) NULL DEFAULT NULL COMMENT '更新人', + `update_time` datetime DEFAULT CURRENT_TIMESTAMP COMMENT 'update time', + PRIMARY KEY (`id`), + UNIQUE INDEX `uniq_com_label_kid` (`combined_label_value`, `key_id`) ); \ No newline at end of file diff --git a/pom.xml b/pom.xml index a0e14f0fb4..d9e16d9e17 100644 --- a/pom.xml +++ b/pom.xml @@ -949,6 +949,14 @@ test + + + org.instancio + instancio-junit + 2.16.1 + test + + org.mockito mockito-core From 2f0ccd870a484740dc4c7c4ecf5b856649095811 Mon Sep 17 00:00:00 2001 From: casionone Date: Sun, 25 Jun 2023 20:34:05 +0800 Subject: [PATCH 173/261] code optimize --- .../restful/api/ConfigurationRestfulApi.java | 17 ++++++++++++++++- .../impl/TemplateConfigKeyServiceImpl.java | 14 ++++++-------- 2 files changed, 22 insertions(+), 9 deletions(-) diff --git a/linkis-public-enhancements/linkis-configuration/src/main/java/org/apache/linkis/configuration/restful/api/ConfigurationRestfulApi.java b/linkis-public-enhancements/linkis-configuration/src/main/java/org/apache/linkis/configuration/restful/api/ConfigurationRestfulApi.java index 8d4a9a3455..f33b4d6f75 100644 --- a/linkis-public-enhancements/linkis-configuration/src/main/java/org/apache/linkis/configuration/restful/api/ConfigurationRestfulApi.java +++ b/linkis-public-enhancements/linkis-configuration/src/main/java/org/apache/linkis/configuration/restful/api/ConfigurationRestfulApi.java @@ -43,6 +43,7 @@ import java.io.IOException; import java.text.MessageFormat; import java.util.ArrayList; +import java.util.HashMap; import java.util.List; import java.util.Map; @@ -167,7 +168,21 @@ public Message getItemList( String userName = ModuleUserUtils.getOperationUser(req, "getItemList with engineType:" + engineType); List result = configKeyService.getConfigKeyList(engineType); - return Message.ok().data("itemList", result); + List filterResult=new ArrayList<>(); + for (ConfigKey configKey : result) { + Map temp =new HashMap(); + temp.put("",configKey.getBoundaryType()); + temp.put("key", configKey.getKey()); + temp.put("name", configKey.getName()); + temp.put("description", configKey.getDescription()); + temp.put("engineType", configKey.getEngineType()); + temp.put("validateType", configKey.getValidateType()); + temp.put("validateRange", configKey.getValidateRange()); + temp.put("boundaryType",configKey.getBoundaryType()); + filterResult.add(temp); + } + + return Message.ok().data("itemList", filterResult); } diff --git a/linkis-public-enhancements/linkis-configuration/src/main/java/org/apache/linkis/configuration/service/impl/TemplateConfigKeyServiceImpl.java b/linkis-public-enhancements/linkis-configuration/src/main/java/org/apache/linkis/configuration/service/impl/TemplateConfigKeyServiceImpl.java index 4c65f2d949..a39130ce3f 100644 --- a/linkis-public-enhancements/linkis-configuration/src/main/java/org/apache/linkis/configuration/service/impl/TemplateConfigKeyServiceImpl.java +++ b/linkis-public-enhancements/linkis-configuration/src/main/java/org/apache/linkis/configuration/service/impl/TemplateConfigKeyServiceImpl.java @@ -36,10 +36,7 @@ import org.springframework.transaction.annotation.Transactional; import java.text.MessageFormat; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; +import java.util.*; import java.util.stream.Collectors; @Service @@ -85,11 +82,12 @@ public Boolean updateKeyMapping( // 待更新的key id 列表 List keyIdList = configKeyList.stream().map(e -> e.getId()).collect(Collectors.toList()); if (configKeyList.size() != itemList.size()) { + List dbKeyList = configKeyList.stream().map(e -> e.getKey()).collect(Collectors.toList()); String msg = MessageFormat.format( "The num of config item data from the DB is inconsistent(DB中获取到的配置数据条数不一致) :" - + "engineType:{0}, input keyList size:{1}, db keyList size:{2}", - engineType, keyList.size(), configKeyList.size()); + + "engineType:{0}, input keyList:{1}, db keyList:{2}.", + engineType, Arrays.toString(keyList.toArray()), Arrays.toString(dbKeyList.toArray())); throw new ConfigurationException(msg); } // 组装更新 @@ -114,7 +112,7 @@ public Boolean updateKeyMapping( String msg = MessageFormat.format( "Parameter configValue verification failed(参数configValue校验失败):" - + "key:{0}, ValidateType:{1}, ValidateRange:{},ConfigValue:{}", + + "key:{0}, ValidateType:{1}, ValidateRange:{2},ConfigValue:{3}", key, validateType, validateRange, configValue); throw new ConfigurationException(msg); } @@ -122,7 +120,7 @@ public Boolean updateKeyMapping( String msg = MessageFormat.format( "Parameter maxValue verification failed(参数maxValue校验失败):" - + "key:{0}, ValidateType:{1}, ValidateRange:{},ConfigValue:{}", + + "key:{0}, ValidateType:{1}, ValidateRange:{2},ConfigValue:{3}", key, validateType, validateRange, maxValue); throw new ConfigurationException(msg); } From 834d06d903a3523bde48cb7365e96170e2de229b Mon Sep 17 00:00:00 2001 From: casionone Date: Sun, 25 Jun 2023 20:37:16 +0800 Subject: [PATCH 174/261] fix bug and code optimize --- .../dao/ConfigKeyLimitForUserMapper.java | 4 +- .../dao/TemplateConfigKeyMapper.java | 8 +- .../configuration/dao/UserTenantMapper.java | 6 +- .../configuration/entity/ConfigKey.java | 51 +- .../entity/ConfigKeyLimitForUser.java | 2 +- .../entity/TemplateConfigKey.java | 2 +- .../service/TemplateConfigKeyService.java | 24 +- .../service/TenantConfigService.java | 2 +- .../service/UserIpConfigService.java | 2 +- .../impl/TemplateConfigKeyServiceImpl.java | 596 +++++++++--------- .../common/ConfigKeyLimitForUserMapper.xml | 26 +- .../resources/mapper/common/ConfigMapper.xml | 8 +- .../mapper/common/TemplateConfigKeyMapper.xml | 32 +- .../service/ConfigurationService.scala | 1 - .../dao/ConfigKeyLimitForUserMapperTest.java | 88 ++- .../configuration/dao/ConfigMapperTest.java | 1 + .../dao/TemplateConfigKeyMapperTest.java | 131 ++-- .../api/ConfigurationRestfulApiTest.java | 2 +- .../src/test/resources/create.sql | 10 +- .../src/test/resources/data.sql | 14 +- 20 files changed, 537 insertions(+), 473 deletions(-) diff --git a/linkis-public-enhancements/linkis-configuration/src/main/java/org/apache/linkis/configuration/dao/ConfigKeyLimitForUserMapper.java b/linkis-public-enhancements/linkis-configuration/src/main/java/org/apache/linkis/configuration/dao/ConfigKeyLimitForUserMapper.java index c9f259d661..fe3077ea3b 100644 --- a/linkis-public-enhancements/linkis-configuration/src/main/java/org/apache/linkis/configuration/dao/ConfigKeyLimitForUserMapper.java +++ b/linkis-public-enhancements/linkis-configuration/src/main/java/org/apache/linkis/configuration/dao/ConfigKeyLimitForUserMapper.java @@ -24,7 +24,7 @@ import java.util.List; /** - * config_key_limit_for_user表的dao接口类 @Description + * linkis_ps_configuration_key_limit_for_user表的dao接口类 @Description * * @version 1.0 */ @@ -86,7 +86,7 @@ public interface ConfigKeyLimitForUserMapper { * @param pageSize */ List selectListByPage( - @Param("id") Long id, @Param("pageNo") int pageNo, @Param("pageSize") int pageSize); + @Param("id") Long id, @Param("pageNo") int pageNo, @Param("pageSize") int pageSize); /** * 动态字段,根据主键来更新符合条件的数据库记录 diff --git a/linkis-public-enhancements/linkis-configuration/src/main/java/org/apache/linkis/configuration/dao/TemplateConfigKeyMapper.java b/linkis-public-enhancements/linkis-configuration/src/main/java/org/apache/linkis/configuration/dao/TemplateConfigKeyMapper.java index cec10fbc24..cb9bd44725 100644 --- a/linkis-public-enhancements/linkis-configuration/src/main/java/org/apache/linkis/configuration/dao/TemplateConfigKeyMapper.java +++ b/linkis-public-enhancements/linkis-configuration/src/main/java/org/apache/linkis/configuration/dao/TemplateConfigKeyMapper.java @@ -24,7 +24,7 @@ import java.util.List; /** - * template_config_key表的dao接口类 @Description + * linkis_ps_configuration_template_config_key表的dao接口类 @Description * * @version 1.0 */ @@ -85,7 +85,7 @@ public interface TemplateConfigKeyMapper { * @param pageSize */ List selectListByPage( - @Param("id") Long id, @Param("pageNo") int pageNo, @Param("pageSize") int pageSize); + @Param("id") Long id, @Param("pageNo") int pageNo, @Param("pageSize") int pageSize); /** * 动态字段,根据主键来更新符合条件的数据库记录 @@ -105,10 +105,10 @@ List selectListByPage( List selectListByTemplateUuid(@Param("templateUuid") String templateUuid); int deleteByTemplateUuidAndKeyIdList( - @Param("templateUuid") String templateUuid, @Param("KeyIdList") List KeyIdList); + @Param("templateUuid") String templateUuid, @Param("KeyIdList") List KeyIdList); int batchInsertOrUpdateList(List list); List selectListByTemplateUuidList( - @Param("templateUuidList") List templateUuidList); + @Param("templateUuidList") List templateUuidList); } diff --git a/linkis-public-enhancements/linkis-configuration/src/main/java/org/apache/linkis/configuration/dao/UserTenantMapper.java b/linkis-public-enhancements/linkis-configuration/src/main/java/org/apache/linkis/configuration/dao/UserTenantMapper.java index c141738ced..b878073610 100644 --- a/linkis-public-enhancements/linkis-configuration/src/main/java/org/apache/linkis/configuration/dao/UserTenantMapper.java +++ b/linkis-public-enhancements/linkis-configuration/src/main/java/org/apache/linkis/configuration/dao/UserTenantMapper.java @@ -26,9 +26,9 @@ public interface UserTenantMapper { List queryTenantList( - @Param("user") String user, - @Param("creator") String creator, - @Param("tenant_value") String tenant); + @Param("user") String user, + @Param("creator") String creator, + @Param("tenant_value") String tenant); void deleteTenant(@Param("id") Integer id); diff --git a/linkis-public-enhancements/linkis-configuration/src/main/java/org/apache/linkis/configuration/entity/ConfigKey.java b/linkis-public-enhancements/linkis-configuration/src/main/java/org/apache/linkis/configuration/entity/ConfigKey.java index 384259a342..5e1cb0b8ca 100644 --- a/linkis-public-enhancements/linkis-configuration/src/main/java/org/apache/linkis/configuration/entity/ConfigKey.java +++ b/linkis-public-enhancements/linkis-configuration/src/main/java/org/apache/linkis/configuration/entity/ConfigKey.java @@ -157,20 +157,41 @@ public void setBoundaryType(Integer boundaryType) { @Override public String toString() { - return "ConfigKey{" + - "id=" + id + - ", key='" + key + '\'' + - ", description='" + description + '\'' + - ", name='" + name + '\'' + - ", engineType='" + engineType + '\'' + - ", defaultValue='" + defaultValue + '\'' + - ", validateType='" + validateType + '\'' + - ", validateRange='" + validateRange + '\'' + - ", isAdvanced=" + isAdvanced + - ", isHidden=" + isHidden + - ", level=" + level + - ", treeName='" + treeName + '\'' + - ", boundaryType=" + boundaryType + - '}'; + return "ConfigKey{" + + "id=" + + id + + ", key='" + + key + + '\'' + + ", description='" + + description + + '\'' + + ", name='" + + name + + '\'' + + ", engineType='" + + engineType + + '\'' + + ", defaultValue='" + + defaultValue + + '\'' + + ", validateType='" + + validateType + + '\'' + + ", validateRange='" + + validateRange + + '\'' + + ", isAdvanced=" + + isAdvanced + + ", isHidden=" + + isHidden + + ", level=" + + level + + ", treeName='" + + treeName + + '\'' + + ", boundaryType=" + + boundaryType + + '}'; } } diff --git a/linkis-public-enhancements/linkis-configuration/src/main/java/org/apache/linkis/configuration/entity/ConfigKeyLimitForUser.java b/linkis-public-enhancements/linkis-configuration/src/main/java/org/apache/linkis/configuration/entity/ConfigKeyLimitForUser.java index 8fd64dff7e..405018ab65 100644 --- a/linkis-public-enhancements/linkis-configuration/src/main/java/org/apache/linkis/configuration/entity/ConfigKeyLimitForUser.java +++ b/linkis-public-enhancements/linkis-configuration/src/main/java/org/apache/linkis/configuration/entity/ConfigKeyLimitForUser.java @@ -20,7 +20,7 @@ import java.util.Date; /** - * config_key_limit_for_user表的entity类 @Description + * linkis_ps_configuration_key_limit_for_user表的entity类 @Description * * @version 1.0 * @author webank diff --git a/linkis-public-enhancements/linkis-configuration/src/main/java/org/apache/linkis/configuration/entity/TemplateConfigKey.java b/linkis-public-enhancements/linkis-configuration/src/main/java/org/apache/linkis/configuration/entity/TemplateConfigKey.java index a89dbd5fa7..93efefd74d 100644 --- a/linkis-public-enhancements/linkis-configuration/src/main/java/org/apache/linkis/configuration/entity/TemplateConfigKey.java +++ b/linkis-public-enhancements/linkis-configuration/src/main/java/org/apache/linkis/configuration/entity/TemplateConfigKey.java @@ -20,7 +20,7 @@ import java.util.Date; /** - * template_config_key表的entity类 @Description + * linkis_ps_configuration_template_config_key表的entity类 @Description * * @version 1.0 * @author webank diff --git a/linkis-public-enhancements/linkis-configuration/src/main/java/org/apache/linkis/configuration/service/TemplateConfigKeyService.java b/linkis-public-enhancements/linkis-configuration/src/main/java/org/apache/linkis/configuration/service/TemplateConfigKeyService.java index 1bd60a7896..3b5e30cc93 100644 --- a/linkis-public-enhancements/linkis-configuration/src/main/java/org/apache/linkis/configuration/service/TemplateConfigKeyService.java +++ b/linkis-public-enhancements/linkis-configuration/src/main/java/org/apache/linkis/configuration/service/TemplateConfigKeyService.java @@ -26,22 +26,22 @@ public interface TemplateConfigKeyService { Boolean updateKeyMapping( - String templateUid, - String templateName, - String engineType, - String operator, - List itemList, - Boolean isFullMode) + String templateUid, + String templateName, + String engineType, + String operator, + List itemList, + Boolean isFullMode) throws ConfigurationException; List queryKeyInfoList(List uuidList) throws ConfigurationException; Map apply( - String templateUid, - String application, - String engineType, - String engineVersion, - String operator, - List userList) + String templateUid, + String application, + String engineType, + String engineVersion, + String operator, + List userList) throws ConfigurationException; } diff --git a/linkis-public-enhancements/linkis-configuration/src/main/java/org/apache/linkis/configuration/service/TenantConfigService.java b/linkis-public-enhancements/linkis-configuration/src/main/java/org/apache/linkis/configuration/service/TenantConfigService.java index 87b14a9c5e..deb9541dbd 100644 --- a/linkis-public-enhancements/linkis-configuration/src/main/java/org/apache/linkis/configuration/service/TenantConfigService.java +++ b/linkis-public-enhancements/linkis-configuration/src/main/java/org/apache/linkis/configuration/service/TenantConfigService.java @@ -25,7 +25,7 @@ public interface TenantConfigService { Map queryTenantList( - String user, String creator, String tenantValue, Integer pageNow, Integer pageSize); + String user, String creator, String tenantValue, Integer pageNow, Integer pageSize); void deleteTenant(Integer id) throws ConfigurationException; diff --git a/linkis-public-enhancements/linkis-configuration/src/main/java/org/apache/linkis/configuration/service/UserIpConfigService.java b/linkis-public-enhancements/linkis-configuration/src/main/java/org/apache/linkis/configuration/service/UserIpConfigService.java index 6d292d6813..9e49b53731 100644 --- a/linkis-public-enhancements/linkis-configuration/src/main/java/org/apache/linkis/configuration/service/UserIpConfigService.java +++ b/linkis-public-enhancements/linkis-configuration/src/main/java/org/apache/linkis/configuration/service/UserIpConfigService.java @@ -31,7 +31,7 @@ public interface UserIpConfigService { void deleteUserIP(Integer id) throws ConfigurationException; Map queryUserIPList( - String user, String creator, Integer pageNow, Integer pageSize); + String user, String creator, Integer pageNow, Integer pageSize); boolean userExists(String user, String creator); diff --git a/linkis-public-enhancements/linkis-configuration/src/main/java/org/apache/linkis/configuration/service/impl/TemplateConfigKeyServiceImpl.java b/linkis-public-enhancements/linkis-configuration/src/main/java/org/apache/linkis/configuration/service/impl/TemplateConfigKeyServiceImpl.java index a39130ce3f..20f6499f25 100644 --- a/linkis-public-enhancements/linkis-configuration/src/main/java/org/apache/linkis/configuration/service/impl/TemplateConfigKeyServiceImpl.java +++ b/linkis-public-enhancements/linkis-configuration/src/main/java/org/apache/linkis/configuration/service/impl/TemplateConfigKeyServiceImpl.java @@ -29,323 +29,321 @@ import org.apache.linkis.configuration.validate.ValidatorManager; import org.apache.linkis.manager.label.builder.CombinedLabelBuilder; import org.apache.linkis.manager.label.entity.CombinedLabel; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; + import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; import org.springframework.transaction.annotation.Transactional; import java.text.MessageFormat; -import java.util.*; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; import java.util.stream.Collectors; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + @Service public class TemplateConfigKeyServiceImpl implements TemplateConfigKeyService { - private static final Logger logger = LoggerFactory.getLogger(TemplateConfigKeyServiceImpl.class); - - @Autowired - private ConfigMapper configMapper; - - @Autowired - private LabelMapper labelMapper; - - @Autowired - private TemplateConfigKeyMapper templateConfigKeyMapper; - - @Autowired - private ConfigurationService configurationService; - - @Autowired - private ValidatorManager validatorManager; - - @Autowired - private ConfigKeyLimitForUserMapper configKeyLimitForUserMapper; - - private CombinedLabelBuilder combinedLabelBuilder = new CombinedLabelBuilder(); - - @Override - @Transactional - public Boolean updateKeyMapping( - String templateUid, - String templateName, - String engineType, - String operator, - List itemList, - Boolean isFullMode) - throws ConfigurationException { - // isFullMode true - // 查询对应的数据 并做数据合法性检查 - List keyList = itemList.stream().map(e -> e.getKey()).collect(Collectors.toList()); - List configKeyList = - configMapper.selectKeyByEngineTypeAndKeyList(engineType, keyList); - // 待更新的key id 列表 - List keyIdList = configKeyList.stream().map(e -> e.getId()).collect(Collectors.toList()); - if (configKeyList.size() != itemList.size()) { - List dbKeyList = configKeyList.stream().map(e -> e.getKey()).collect(Collectors.toList()); - String msg = - MessageFormat.format( - "The num of config item data from the DB is inconsistent(DB中获取到的配置数据条数不一致) :" - + "engineType:{0}, input keyList:{1}, db keyList:{2}.", - engineType, Arrays.toString(keyList.toArray()), Arrays.toString(dbKeyList.toArray())); - throw new ConfigurationException(msg); - } - // 组装更新 - List toUpdateOrInsertList = new ArrayList<>(); - - // map k:v---> key：ConfigKey - Map configKeyMap = - configKeyList.stream().collect(Collectors.toMap(ConfigKey::getKey, item -> item)); - for (TemplateConfigKeyVo item : itemList) { - - String key = item.getKey(); - ConfigKey temp = configKeyMap.get(item.getKey()); - Long keyId = temp.getId(); - String validateType = temp.getValidateType(); - String validateRange = temp.getValidateRange(); - String configValue = item.getConfigValue(); - String maxValue = item.getMaxValue(); - - if (!validatorManager - .getOrCreateValidator(validateType) - .validate(configValue, validateRange)) { - String msg = - MessageFormat.format( - "Parameter configValue verification failed(参数configValue校验失败):" - + "key:{0}, ValidateType:{1}, ValidateRange:{2},ConfigValue:{3}", - key, validateType, validateRange, configValue); - throw new ConfigurationException(msg); - } - if (!validatorManager.getOrCreateValidator(validateType).validate(maxValue, validateRange)) { - String msg = - MessageFormat.format( - "Parameter maxValue verification failed(参数maxValue校验失败):" - + "key:{0}, ValidateType:{1}, ValidateRange:{2},ConfigValue:{3}", - key, validateType, validateRange, maxValue); - throw new ConfigurationException(msg); - } - - TemplateConfigKey templateConfigKey = new TemplateConfigKey(); - templateConfigKey.setTemplateName(templateName); - templateConfigKey.setTemplateUuid(templateUid); - templateConfigKey.setKeyId(keyId); - templateConfigKey.setConfigValue(configValue); - templateConfigKey.setMaxValue(maxValue); - templateConfigKey.setCreateBy(operator); - templateConfigKey.setUpdateBy(operator); - toUpdateOrInsertList.add(templateConfigKey); - } - // 根据不同模式更新数据 - if (isFullMode) { - // 之前在数据库中的数据 需要移除的 - List oldList = - templateConfigKeyMapper.selectListByTemplateUuid(templateUid); - List needToRemoveList = - oldList.stream() - .filter( - item -> { - return !keyIdList.contains(item.getKeyId()); - }) - .map(e -> e.getKeyId()) - .collect(Collectors.toList()); - logger.info( - "Try to remove old data:[" + needToRemoveList + "] for templateUid:" + templateUid); - templateConfigKeyMapper.deleteByTemplateUuidAndKeyIdList(templateUid, needToRemoveList); - } - - templateConfigKeyMapper.batchInsertOrUpdateList(toUpdateOrInsertList); - - return true; + private static final Logger logger = LoggerFactory.getLogger(TemplateConfigKeyServiceImpl.class); + + @Autowired private ConfigMapper configMapper; + + @Autowired private LabelMapper labelMapper; + + @Autowired private TemplateConfigKeyMapper templateConfigKeyMapper; + + @Autowired private ConfigurationService configurationService; + + @Autowired private ValidatorManager validatorManager; + + @Autowired private ConfigKeyLimitForUserMapper configKeyLimitForUserMapper; + + private CombinedLabelBuilder combinedLabelBuilder = new CombinedLabelBuilder(); + + @Override + @Transactional + public Boolean updateKeyMapping( + String templateUid, + String templateName, + String engineType, + String operator, + List itemList, + Boolean isFullMode) + throws ConfigurationException { + // isFullMode true + // 查询对应的数据 并做数据合法性检查 + List keyList = itemList.stream().map(e -> e.getKey()).collect(Collectors.toList()); + List configKeyList = + configMapper.selectKeyByEngineTypeAndKeyList(engineType, keyList); + // 待更新的key id 列表 + List keyIdList = configKeyList.stream().map(e -> e.getId()).collect(Collectors.toList()); + if (configKeyList.size() != itemList.size()) { + String msg = + MessageFormat.format( + "The num of config item data from the DB is inconsistent(DB中获取到的配置数据条数不一致) :" + + "engineType:{0}, input keyList size:{1}, db keyList size:{2}", + engineType, keyList.size(), configKeyList.size()); + throw new ConfigurationException(msg); + } + // 组装更新 + List toUpdateOrInsertList = new ArrayList<>(); + + // map k:v---> key：ConfigKey + Map configKeyMap = + configKeyList.stream().collect(Collectors.toMap(ConfigKey::getKey, item -> item)); + for (TemplateConfigKeyVo item : itemList) { + + String key = item.getKey(); + ConfigKey temp = configKeyMap.get(item.getKey()); + Long keyId = temp.getId(); + String validateType = temp.getValidateType(); + String validateRange = temp.getValidateRange(); + String configValue = item.getConfigValue(); + String maxValue = item.getMaxValue(); + + if (!validatorManager + .getOrCreateValidator(validateType) + .validate(configValue, validateRange)) { + String msg = + MessageFormat.format( + "Parameter configValue verification failed(参数configValue校验失败):" + + "key:{0}, ValidateType:{1}, ValidateRange:{3},ConfigValue:{4}", + key, validateType, validateRange, configValue); + throw new ConfigurationException(msg); + } + if (!validatorManager.getOrCreateValidator(validateType).validate(maxValue, validateRange)) { + String msg = + MessageFormat.format( + "Parameter maxValue verification failed(参数maxValue校验失败):" + + "key:{0}, ValidateType:{1}, ValidateRange:{3},ConfigValue:{4}", + key, validateType, validateRange, maxValue); + throw new ConfigurationException(msg); + } + + TemplateConfigKey templateConfigKey = new TemplateConfigKey(); + templateConfigKey.setTemplateName(templateName); + templateConfigKey.setTemplateUuid(templateUid); + templateConfigKey.setKeyId(keyId); + templateConfigKey.setConfigValue(configValue); + templateConfigKey.setMaxValue(maxValue); + templateConfigKey.setCreateBy(operator); + templateConfigKey.setUpdateBy(operator); + toUpdateOrInsertList.add(templateConfigKey); + } + // 根据不同模式更新数据 + if (isFullMode) { + // 之前在数据库中的数据 需要移除的 + List oldList = + templateConfigKeyMapper.selectListByTemplateUuid(templateUid); + List needToRemoveList = + oldList.stream() + .filter( + item -> { + return !keyIdList.contains(item.getKeyId()); + }) + .map(e -> e.getKeyId()) + .collect(Collectors.toList()); + logger.info( + "Try to remove old data:[" + needToRemoveList + "] for templateUid:" + templateUid); + templateConfigKeyMapper.deleteByTemplateUuidAndKeyIdList(templateUid, needToRemoveList); } - @Override - public List queryKeyInfoList(List uuidList) throws ConfigurationException { - List result = new ArrayList<>(); - - List templateConfigKeyList = - templateConfigKeyMapper.selectListByTemplateUuidList(uuidList); - - Map> templateConfigKeyListGroupByUuid = - templateConfigKeyList.stream() - .collect(Collectors.groupingBy(TemplateConfigKey::getTemplateUuid)); - - List keyIdList = - templateConfigKeyList.stream().map(e -> e.getId()).distinct().collect(Collectors.toList()); - List configKeyList = configMapper.selectKeyByKeyIdList(keyIdList); - // map k:v---> keyId：ConfigKey - Map configKeyMap = - configKeyList.stream().collect(Collectors.toMap(ConfigKey::getId, item -> item)); - - for (String uuid : templateConfigKeyListGroupByUuid.keySet()) { - Map item = new HashMap(); - List keys = new ArrayList<>(); - item.put("templateUid", uuid); - - List group = templateConfigKeyListGroupByUuid.get(uuid); - for (TemplateConfigKey templateConfigKey : group) { - // Map temp = BDPJettyServerHelper.jacksonJson().convertValue(templateConfigKey, Map.class); - Map temp = new HashMap(); - - temp.put("configValue", templateConfigKey.getConfigValue()); - temp.put("maxValue", templateConfigKey.getMaxValue()); - temp.put("createBy", templateConfigKey.getCreateBy()); - temp.put("createTime", templateConfigKey.getCreateTime()); - temp.put("updateBy", templateConfigKey.getUpdateBy()); - temp.put("updateTime", templateConfigKey.getUpdateTime()); - temp.put("keyId", templateConfigKey.getKeyId()); - - ConfigKey info = configKeyMap.get(templateConfigKey.getKeyId()); - temp.put("key", info.getKey()); - temp.put("name", info.getName()); - temp.put("description", info.getDescription()); - temp.put("engineType", info.getEngineType()); - temp.put("validateType", info.getValidateType()); - temp.put("validateRange", info.getValidateRange()); - temp.put("boundaryType",info.getBoundaryType()); - - keys.add(temp); - } - item.put("itemList", keys); - result.add(item); - } - return result; + templateConfigKeyMapper.batchInsertOrUpdateList(toUpdateOrInsertList); + + return true; + } + + @Override + public List queryKeyInfoList(List uuidList) throws ConfigurationException { + List result = new ArrayList<>(); + + List templateConfigKeyList = + templateConfigKeyMapper.selectListByTemplateUuidList(uuidList); + + Map> templateConfigKeyListGroupByUuid = + templateConfigKeyList.stream() + .collect(Collectors.groupingBy(TemplateConfigKey::getTemplateUuid)); + + List keyIdList = + templateConfigKeyList.stream().map(e -> e.getId()).distinct().collect(Collectors.toList()); + List configKeyList = configMapper.selectKeyByKeyIdList(keyIdList); + // map k:v---> keyId：ConfigKey + Map configKeyMap = + configKeyList.stream().collect(Collectors.toMap(ConfigKey::getId, item -> item)); + + for (String uuid : templateConfigKeyListGroupByUuid.keySet()) { + Map item = new HashMap(); + List keys = new ArrayList<>(); + item.put("templateUid", uuid); + + List group = templateConfigKeyListGroupByUuid.get(uuid); + for (TemplateConfigKey templateConfigKey : group) { + // Map temp = BDPJettyServerHelper.jacksonJson().convertValue(templateConfigKey, Map.class); + Map temp = new HashMap(); + + temp.put("configValue", templateConfigKey.getConfigValue()); + temp.put("maxValue", templateConfigKey.getMaxValue()); + temp.put("createBy", templateConfigKey.getCreateBy()); + temp.put("createTime", templateConfigKey.getCreateTime()); + temp.put("updateBy", templateConfigKey.getUpdateBy()); + temp.put("updateTime", templateConfigKey.getUpdateTime()); + temp.put("keyId", templateConfigKey.getKeyId()); + + ConfigKey info = configKeyMap.get(templateConfigKey.getKeyId()); + temp.put("key", info.getKey()); + temp.put("name", info.getName()); + temp.put("description", info.getDescription()); + temp.put("engineType", info.getEngineType()); + temp.put("validateType", info.getValidateType()); + temp.put("validateRange", info.getValidateRange()); + temp.put("boundaryType", info.getBoundaryType()); + + keys.add(temp); + } + item.put("itemList", keys); + result.add(item); } + return result; + } + + // @Override + // public List apply1(String templateUid, String application, String engineType, + // String engineVersion, String operator, List userList) + // throws ConfigurationException { + // // 确认用户该标签是否都存在 + // // combined_label_value 组合标签 combined_userCreator_engineType 如 + // hadoop-IDE,spark-2.4.3 + // String combinedKey = LabelEntityParser.COMBINED_USERCREATOR_ENGINETYPE; + // String valueFormat = "{}-{},{}-{}"; + // + // List configLabelList = userList.stream().filter(e -> + // StringUtils.isNotBlank(e)) + // .map(user -> { + //// ConfigLabel label = new ConfigLabel(); + //// label.setLabelKey(combinedKey); + //// String + // value=MessageFormat.format(valueFormat,user,application,engineType,engineVersion); + //// label.setStringValue(value); + //// label.setFeature(Feature.OPTIONAL); + //// label.setLabelValueSize(); + // CombinedLabel combinedLabel = + // configurationService.generateCombinedLabel(engineType, + // engineVersion, user, application); + // ConfigLabel configLabel = + // LabelEntityParser.parseToConfigLabel(combinedLabel); + // return configLabel; + // }) + // .collect(Collectors.toList()); + // // 尝试批量创建用户label信息 + // labelMapper.batchInsertLabel(configLabelList); + // List valueList = configLabelList.stream().map(e -> + // e.getStringValue()).collect(Collectors.toList()); + // List labelList = + // labelMapper.selectUserCreatorEngineTypeLabelList(valueList); + // + // //组装批量更新的configvalue + // List needUpdataList = new ArrayList<>(); + // + // ConfigValue configValue = null; + // configValue = new ConfigValue(); + // configValue.setConfigKeyId(configKey.getId()); + // configValue.setConfigValue(configKeyValue.getConfigValue()); + // configValue.setConfigLabelId(configLabel.getId()); + // //获取label id + // + // //更新对应的配置值 + // + // } + + @Override + public Map apply( + String templateUid, + String application, + String engineType, + String engineVersion, + String operator, + List userList) + throws ConfigurationException { + List successList = new ArrayList<>(); + List errorList = new ArrayList<>(); + + // get the associated config itsm list + List templateUuidList = new ArrayList<>(); + templateUuidList.add(templateUid); + List templateConfigKeyList = + templateConfigKeyMapper.selectListByTemplateUuidList(templateUuidList); + + for (String user : userList) { + // try to create combined_userCreator_engineType label for user + Map res = new HashMap(); + res.put("user", user); + try { + CombinedLabel combinedLabel = + configurationService.generateCombinedLabel( + engineType, engineVersion, user, application); + String conbinedLabelKey = combinedLabel.getLabelKey(); + String conbinedLabelStringValue = combinedLabel.getStringValue(); + ConfigLabel configLabel = + labelMapper.getLabelByKeyValue(conbinedLabelKey, conbinedLabelStringValue); + if (null == configLabel || configLabel.getId() < 0) { + configLabel = LabelEntityParser.parseToConfigLabel(combinedLabel); + labelMapper.insertLabel(configLabel); + logger.info("succeed to create label: {}", configLabel.getStringValue()); + } - // @Override - // public List apply1(String templateUid, String application, String engineType, - // String engineVersion, String operator, List userList) - // throws ConfigurationException { - // // 确认用户该标签是否都存在 - // // combined_label_value 组合标签 combined_userCreator_engineType 如 - // hadoop-IDE,spark-2.4.3 - // String combinedKey = LabelEntityParser.COMBINED_USERCREATOR_ENGINETYPE; - // String valueFormat = "{}-{},{}-{}"; - // - // List configLabelList = userList.stream().filter(e -> - // StringUtils.isNotBlank(e)) - // .map(user -> { - //// ConfigLabel label = new ConfigLabel(); - //// label.setLabelKey(combinedKey); - //// String - // value=MessageFormat.format(valueFormat,user,application,engineType,engineVersion); - //// label.setStringValue(value); - //// label.setFeature(Feature.OPTIONAL); - //// label.setLabelValueSize(); - // CombinedLabel combinedLabel = - // configurationService.generateCombinedLabel(engineType, - // engineVersion, user, application); - // ConfigLabel configLabel = - // LabelEntityParser.parseToConfigLabel(combinedLabel); - // return configLabel; - // }) - // .collect(Collectors.toList()); - // // 尝试批量创建用户label信息 - // labelMapper.batchInsertLabel(configLabelList); - // List valueList = configLabelList.stream().map(e -> - // e.getStringValue()).collect(Collectors.toList()); - // List labelList = - // labelMapper.selectUserCreatorEngineTypeLabelList(valueList); - // - // //组装批量更新的configvalue - // List needUpdataList = new ArrayList<>(); - // - // ConfigValue configValue = null; - // configValue = new ConfigValue(); - // configValue.setConfigKeyId(configKey.getId()); - // configValue.setConfigValue(configKeyValue.getConfigValue()); - // configValue.setConfigLabelId(configLabel.getId()); - // //获取label id - // - // //更新对应的配置值 - // - // } - - @Override - public Map apply( - String templateUid, - String application, - String engineType, - String engineVersion, - String operator, - List userList) - throws ConfigurationException { - List successList = new ArrayList<>(); - List errorList = new ArrayList<>(); - - // get the associated config itsm list - List templateUuidList = new ArrayList<>(); - templateUuidList.add(templateUid); - List templateConfigKeyList = - templateConfigKeyMapper.selectListByTemplateUuidList(templateUuidList); - - for (String user : userList) { - // try to create combined_userCreator_engineType label for user - Map res = new HashMap(); - res.put("user", user); - try { - CombinedLabel combinedLabel = - configurationService.generateCombinedLabel( - engineType, engineVersion, user, application); - String conbinedLabelKey = combinedLabel.getLabelKey(); - String conbinedLabelStringValue = combinedLabel.getStringValue(); - ConfigLabel configLabel = - labelMapper.getLabelByKeyValue(conbinedLabelKey, conbinedLabelStringValue); - if (null == configLabel || configLabel.getId() < 0) { - configLabel = LabelEntityParser.parseToConfigLabel(combinedLabel); - labelMapper.insertLabel(configLabel); - logger.info("succeed to create label: {}", configLabel.getStringValue()); - } - - // batch update config value - List configValues = new ArrayList<>(); - - List configKeyLimitForUsers = new ArrayList<>(); - - for (TemplateConfigKey templateConfigKey : templateConfigKeyList) { - Long keyId = templateConfigKey.getKeyId(); - String uuid = templateConfigKey.getTemplateUuid(); - - ConfigValue configValue = new ConfigValue(); - configValue.setConfigKeyId(keyId); - configValue.setConfigValue(templateConfigKey.getConfigValue()); - configValue.setConfigLabelId(configLabel.getId()); - configValues.add(configValue); - - ConfigKeyLimitForUser configKeyLimitForUser = new ConfigKeyLimitForUser(); - configKeyLimitForUser.setUserName(user); - configKeyLimitForUser.setCombinedLabelValue(configLabel.getStringValue()); - configKeyLimitForUser.setKeyId(keyId); - configKeyLimitForUser.setLatestUpdateTemplateUuid(uuid); - configKeyLimitForUser.setCreateBy(operator); - configKeyLimitForUser.setUpdateBy(operator); - configKeyLimitForUsers.add(configKeyLimitForUser); - } - configMapper.batchInsertValue(configValues); - - // batch update user ConfigKeyLimitForUserMapper - configKeyLimitForUserMapper.batchInsertList(configKeyLimitForUsers); - - successList.add(res); - } catch (Exception e) { - logger.warn("try to update configurations for user:" + user + " with error", e); - res.put("msg", e.getMessage()); - errorList.add(res); - } + // batch update config value + List configValues = new ArrayList<>(); + + List configKeyLimitForUsers = new ArrayList<>(); + + for (TemplateConfigKey templateConfigKey : templateConfigKeyList) { + Long keyId = templateConfigKey.getKeyId(); + String uuid = templateConfigKey.getTemplateUuid(); + + ConfigValue configValue = new ConfigValue(); + configValue.setConfigKeyId(keyId); + configValue.setConfigValue(templateConfigKey.getConfigValue()); + configValue.setConfigLabelId(configLabel.getId()); + configValues.add(configValue); + + ConfigKeyLimitForUser configKeyLimitForUser = new ConfigKeyLimitForUser(); + configKeyLimitForUser.setUserName(user); + configKeyLimitForUser.setCombinedLabelValue(configLabel.getStringValue()); + configKeyLimitForUser.setKeyId(keyId); + configKeyLimitForUser.setLatestUpdateTemplateUuid(uuid); + configKeyLimitForUser.setCreateBy(operator); + configKeyLimitForUser.setUpdateBy(operator); + configKeyLimitForUsers.add(configKeyLimitForUser); } + configMapper.batchInsertValue(configValues); - Map result = new HashMap<>(); + // batch update user ConfigKeyLimitForUserMapper + configKeyLimitForUserMapper.batchInsertList(configKeyLimitForUsers); - Map successResult = new HashMap<>(); - Map errorResult = new HashMap<>(); + successList.add(res); + } catch (Exception e) { + logger.warn("try to update configurations for user:" + user + " with error", e); + res.put("msg", e.getMessage()); + errorList.add(res); + } + } - successResult.put("num", successList.size()); - successResult.put("infoList", successList); + Map result = new HashMap<>(); - errorResult.put("num", errorList.size()); - errorResult.put("infoList", errorList); + Map successResult = new HashMap<>(); + Map errorResult = new HashMap<>(); - result.put("success", successResult); - result.put("error", errorResult); - return result; - } + successResult.put("num", successList.size()); + successResult.put("infoList", successList); + + errorResult.put("num", errorList.size()); + errorResult.put("infoList", errorList); + + result.put("success", successResult); + result.put("error", errorResult); + return result; + } } diff --git a/linkis-public-enhancements/linkis-configuration/src/main/resources/mapper/common/ConfigKeyLimitForUserMapper.xml b/linkis-public-enhancements/linkis-configuration/src/main/resources/mapper/common/ConfigKeyLimitForUserMapper.xml index d58a0ca361..ec4c6cc7f8 100644 --- a/linkis-public-enhancements/linkis-configuration/src/main/resources/mapper/common/ConfigKeyLimitForUserMapper.xml +++ b/linkis-public-enhancements/linkis-configuration/src/main/resources/mapper/common/ConfigKeyLimitForUserMapper.xml @@ -36,14 +36,14 @@ id = #{id,jdbcType=BIGINT} - insert into config_key_limit_for_user (id, user_name, combined_label_value, + insert into linkis_ps_configuration_key_limit_for_user (id, user_name, combined_label_value, key_id, latest_update_template_uuid, is_valid, create_by, create_time, update_by, update_time) @@ -74,11 +74,11 @@ - delete from config_key_limit_for_user + delete from linkis_ps_configuration_key_limit_for_user where id = #{id,jdbcType=BIGINT} - insert into config_key_limit_for_user (id, user_name, combined_label_value, + insert into linkis_ps_configuration_key_limit_for_user (id, user_name, combined_label_value, key_id, latest_update_template_uuid, is_valid, create_by, create_time, update_by, update_time) values (#{id,jdbcType=BIGINT}, #{userName,jdbcType=VARCHAR}, #{combinedLabelValue,jdbcType=VARCHAR}, @@ -86,7 +86,7 @@ #{createBy,jdbcType=VARCHAR}, now(), #{updateBy,jdbcType=VARCHAR}, now()) - insert into config_key_limit_for_user + insert into linkis_ps_configuration_key_limit_for_user id, @@ -145,7 +145,7 @@ - update config_key_limit_for_user + update linkis_ps_configuration_key_limit_for_user user_name = #{userName,jdbcType=VARCHAR}, @@ -173,7 +173,7 @@ where id = #{id,jdbcType=BIGINT} - update config_key_limit_for_user + update linkis_ps_configuration_key_limit_for_user set user_name = #{userName,jdbcType=VARCHAR}, combined_label_value = #{combinedLabelValue,jdbcType=VARCHAR}, key_id = #{keyId,jdbcType=BIGINT}, @@ -188,7 +188,7 @@ - insert into config_key_limit_for_user (user_name, combined_label_value, + insert into linkis_ps_configuration_key_limit_for_user (user_name, combined_label_value, key_id, latest_update_template_uuid, create_by, create_time, update_by, update_time) @@ -205,14 +205,14 @@ now(), - , + #{item.updateBy,jdbcType=VARCHAR}, - #{item.updateTime}, + #{item.updateTime} - now(), + now() ) diff --git a/linkis-public-enhancements/linkis-configuration/src/main/resources/mapper/common/ConfigMapper.xml b/linkis-public-enhancements/linkis-configuration/src/main/resources/mapper/common/ConfigMapper.xml index 9ce1e99e82..5b4526b509 100644 --- a/linkis-public-enhancements/linkis-configuration/src/main/resources/mapper/common/ConfigMapper.xml +++ b/linkis-public-enhancements/linkis-configuration/src/main/resources/mapper/common/ConfigMapper.xml @@ -111,9 +111,13 @@ INSERT INTO linkis_ps_configuration_config_key ( - id, key, description, name, engine_conn_type, default_value, validate_type, validate_range, is_hidden, is_advanced, level, treeName) + id, key, description, name, engine_conn_type, default_value, validate_type, validate_range, is_hidden, is_advanced, level, treeName, boundary_type) VALUES ( - #{id}, #{key}, #{description}, #{name}, #{engineType}, #{defaultValue}, #{validateType}, #{validateRange}, #{isHidden}, #{isAdvanced}, #{level}, #{treeName}) + #{id}, #{key}, #{description}, + #{name}, #{engineType}, #{defaultValue}, + #{validateType}, #{validateRange}, #{isHidden}, + #{isAdvanced}, #{level}, #{treeName}, + #{boundaryType}) select - from template_config_key + from linkis_ps_configuration_template_config_key where id = #{id,jdbcType=BIGINT} id = #{id,jdbcType=BIGINT} - insert into template_config_key (id, template_name, template_uuid, + insert into linkis_ps_configuration_template_config_key (id, template_name, template_uuid, key_id, config_value, max_value, min_value, validate_range, is_valid, create_by, create_time, update_by, update_time @@ -80,11 +80,11 @@ - delete from template_config_key + delete from linkis_ps_configuration_template_config_key where id = #{id,jdbcType=BIGINT} - insert into template_config_key (id, template_name, template_uuid, + insert into linkis_ps_configuration_template_config_key (id, template_name, template_uuid, key_id, config_value, max_value, min_value, validate_range, is_valid, create_by, create_time, update_by, update_time) @@ -94,7 +94,7 @@ #{createBy,jdbcType=VARCHAR}, now(), #{updateBy,jdbcType=VARCHAR}, now()) - insert into template_config_key + insert into linkis_ps_configuration_template_config_key id, @@ -171,7 +171,7 @@ - update template_config_key + update linkis_ps_configuration_template_config_key template_name = #{templateName,jdbcType=VARCHAR}, @@ -208,7 +208,7 @@ where id = #{id,jdbcType=BIGINT} - update template_config_key + update linkis_ps_configuration_template_config_key set template_name = #{templateName,jdbcType=VARCHAR}, template_uuid = #{templateUuid,jdbcType=VARCHAR}, key_id = #{keyId,jdbcType=BIGINT}, @@ -228,14 +228,14 @@ delete - from template_config_key + from linkis_ps_configuration_template_config_key where template_uuid = #{templateUuid,jdbcType=VARCHAR} and key_id in @@ -244,7 +244,7 @@ - insert into template_config_key (template_name, template_uuid, + insert into linkis_ps_configuration_template_config_key (template_name, template_uuid, key_id, config_value, max_value, create_by, create_time, update_by, update_time ) @@ -261,14 +261,14 @@ now(), - , + #{item.updateBy,jdbcType=VARCHAR}, - #{item.updateTime}, + #{item.updateTime} - now(), + now() ) @@ -284,7 +284,7 @@ @@ -190,9 +190,8 @@ @@ -242,11 +241,33 @@ #{id},#{configKeyId},#{configValue}, #{configLabelId}, now(), now()) - - REPLACE INTO linkis_ps_configuration_config_value( - id, config_key_id, config_value, config_label_id, create_time, update_time) - VALUES ( - #{id},#{configKeyId},#{configValue}, #{configLabelId}, now(), now()) + + INSERT INTO linkis_ps_configuration_config_value(config_key_id, config_value, config_label_id, create_time, update_time) + VALUES + + ( + #{item.configKeyId},#{item.configValue}, #{item.configLabelId}, + + + #{item.createTime}, + + + now(), + + + + + #{item.updateTime} + + + now() + + + ) + + on duplicate key update + config_value =values(config_value), + update_time= now() - select - - from linkis_ps_configuration_key_limit_for_user - where id = #{id,jdbcType=BIGINT} - - - id = #{id,jdbcType=BIGINT} - - - + insert into linkis_ps_configuration_key_limit_for_user (id, user_name, combined_label_value, - key_id, latest_update_template_uuid, - is_valid, create_by, create_time, update_by, - update_time) - values + key_id, latest_update_template_uuid, + is_valid, create_by, create_time, update_by, + update_time) + values ( - #{item.id,jdbcType=BIGINT}, #{item.userName,jdbcType=VARCHAR}, #{item.combinedLabelValue,jdbcType=VARCHAR}, - #{item.keyId,jdbcType=BIGINT}, #{item.latestUpdateTemplateUuid,jdbcType=VARCHAR}, - #{item.isValid,jdbcType=VARCHAR}, #{item.createBy,jdbcType=VARCHAR}, now(), #{item.updateBy,jdbcType=VARCHAR}, - now()) + #{item.id,jdbcType=BIGINT}, #{item.userName,jdbcType=VARCHAR}, #{item.combinedLabelValue,jdbcType=VARCHAR}, + #{item.keyId,jdbcType=BIGINT}, #{item.latestUpdateTemplateUuid,jdbcType=VARCHAR}, + #{item.isValid,jdbcType=VARCHAR}, #{item.createBy,jdbcType=VARCHAR}, now(), #{item.updateBy,jdbcType=VARCHAR}, + now()) @@ -79,99 +54,13 @@ insert into linkis_ps_configuration_key_limit_for_user (id, user_name, combined_label_value, - key_id, latest_update_template_uuid, is_valid, - create_by, create_time, update_by, update_time) - values (#{id,jdbcType=BIGINT}, #{userName,jdbcType=VARCHAR}, #{combinedLabelValue,jdbcType=VARCHAR}, - #{keyId,jdbcType=BIGINT}, #{latestUpdateTemplateUuid,jdbcType=VARCHAR}, #{isValid,jdbcType=VARCHAR}, - #{createBy,jdbcType=VARCHAR}, now(), #{updateBy,jdbcType=VARCHAR}, now()) + key_id, latest_update_template_uuid, is_valid, + create_by, create_time, update_by, update_time) + values (#{id,jdbcType=BIGINT}, #{userName,jdbcType=VARCHAR}, #{combinedLabelValue,jdbcType=VARCHAR}, + #{keyId,jdbcType=BIGINT}, #{latestUpdateTemplateUuid,jdbcType=VARCHAR}, #{isValid,jdbcType=VARCHAR}, + #{createBy,jdbcType=VARCHAR}, now(), #{updateBy,jdbcType=VARCHAR}, now()) - - insert into linkis_ps_configuration_key_limit_for_user - - - id, - - - user_name, - - - combined_label_value, - - - key_id, - - - latest_update_template_uuid, - - - is_valid, - - - create_by, - - create_time, - - update_by, - - update_time, - - - - #{id,jdbcType=BIGINT}, - - - #{userName,jdbcType=VARCHAR}, - - - #{combinedLabelValue,jdbcType=VARCHAR}, - - - #{keyId,jdbcType=BIGINT}, - - - #{latestUpdateTemplateUuid,jdbcType=VARCHAR}, - - - #{isValid,jdbcType=VARCHAR}, - - - #{createBy,jdbcType=VARCHAR}, - - now(), - - #{updateBy,jdbcType=VARCHAR}, - - now(), - - - - update linkis_ps_configuration_key_limit_for_user - - - user_name = #{userName,jdbcType=VARCHAR}, - - - combined_label_value = #{combinedLabelValue,jdbcType=VARCHAR}, - - - key_id = #{keyId,jdbcType=BIGINT}, - - - latest_update_template_uuid = #{latestUpdateTemplateUuid,jdbcType=VARCHAR}, - - - is_valid = #{isValid,jdbcType=VARCHAR}, - - - create_by = #{createBy,jdbcType=VARCHAR}, - - - update_by = #{updateBy,jdbcType=VARCHAR}, - - update_time = now(), - - where id = #{id,jdbcType=BIGINT} - + update linkis_ps_configuration_key_limit_for_user set user_name = #{userName,jdbcType=VARCHAR}, @@ -223,4 +112,5 @@ update_time= now() + diff --git a/linkis-public-enhancements/linkis-configuration/src/main/resources/mapper/common/ConfigMapper.xml b/linkis-public-enhancements/linkis-configuration/src/main/resources/mapper/common/ConfigMapper.xml index 8be6b7396e..f385cf81c3 100644 --- a/linkis-public-enhancements/linkis-configuration/src/main/resources/mapper/common/ConfigMapper.xml +++ b/linkis-public-enhancements/linkis-configuration/src/main/resources/mapper/common/ConfigMapper.xml @@ -78,7 +78,7 @@ - `id`, `key`, `description`, `name`, `engine_conn_type`, `default_value`, `validate_type`, `validate_range`, `is_hidden`, `is_advanced`, `level`, `treeName` + `id`, `key`, `description`, `name`, `engine_conn_type`, `default_value`, `validate_type`, `validate_range`, `is_hidden`, `is_advanced`, `level`, `treeName`,`boundary_type` @@ -170,11 +170,27 @@ WHERE `key` = #{keyName} + + - select - - from linkis_ps_configuration_template_config_key - where id = #{id,jdbcType=BIGINT} - - - id = #{id,jdbcType=BIGINT} - - - + insert into linkis_ps_configuration_template_config_key (id, template_name, template_uuid, - key_id, config_value, max_value, - min_value, validate_range, is_valid, - create_by, create_time, update_by, update_time - ) - values + key_id, config_value, max_value, + min_value, validate_range, is_valid, + create_by, create_time, update_by, update_time + ) + values ( - #{item.id,jdbcType=BIGINT}, #{item.templateName,jdbcType=VARCHAR}, #{item.templateUuid,jdbcType=VARCHAR}, - #{item.keyId,jdbcType=BIGINT}, #{item.configValue,jdbcType=VARCHAR}, #{item.maxValue,jdbcType=VARCHAR}, - #{item.minValue,jdbcType=VARCHAR}, #{item.validateRange,jdbcType=VARCHAR}, #{item.isValid,jdbcType=VARCHAR}, - #{item.createBy,jdbcType=VARCHAR}, now(), #{item.updateBy,jdbcType=VARCHAR}, now() - ) + #{item.id,jdbcType=BIGINT}, #{item.templateName,jdbcType=VARCHAR}, #{item.templateUuid,jdbcType=VARCHAR}, + #{item.keyId,jdbcType=BIGINT}, #{item.configValue,jdbcType=VARCHAR}, #{item.maxValue,jdbcType=VARCHAR}, + #{item.minValue,jdbcType=VARCHAR}, #{item.validateRange,jdbcType=VARCHAR}, #{item.isValid,jdbcType=VARCHAR}, + #{item.createBy,jdbcType=VARCHAR}, now(), #{item.updateBy,jdbcType=VARCHAR}, now() + ) - - delete from linkis_ps_configuration_template_config_key - where id = #{id,jdbcType=BIGINT} - - - insert into linkis_ps_configuration_template_config_key (id, template_name, template_uuid, - key_id, config_value, max_value, - min_value, validate_range, is_valid, - create_by, create_time, update_by, update_time) - values (#{id,jdbcType=BIGINT}, #{templateName,jdbcType=VARCHAR}, #{templateUuid,jdbcType=VARCHAR}, - #{keyId,jdbcType=BIGINT}, #{configValue,jdbcType=VARCHAR}, #{maxValue,jdbcType=VARCHAR}, - #{minValue,jdbcType=VARCHAR}, #{validateRange,jdbcType=VARCHAR}, #{isValid,jdbcType=VARCHAR}, - #{createBy,jdbcType=VARCHAR}, now(), #{updateBy,jdbcType=VARCHAR}, now()) - - - insert into linkis_ps_configuration_template_config_key - - - id, - - - template_name, - - - template_uuid, - - - key_id, - - - config_value, - - - max_value, - - - min_value, - - - validate_range, - - - is_valid, - - - create_by, - - create_time, - - update_by, - - update_time, - - - - #{id,jdbcType=BIGINT}, - - - #{templateName,jdbcType=VARCHAR}, - - - #{templateUuid,jdbcType=VARCHAR}, - - - #{keyId,jdbcType=BIGINT}, - - - #{configValue,jdbcType=VARCHAR}, - - - #{maxValue,jdbcType=VARCHAR}, - - - #{minValue,jdbcType=VARCHAR}, - - - #{validateRange,jdbcType=VARCHAR}, - - - #{isValid,jdbcType=VARCHAR}, - - - #{createBy,jdbcType=VARCHAR}, - - now(), - - #{updateBy,jdbcType=VARCHAR}, - - now(), - - - - update linkis_ps_configuration_template_config_key - - - template_name = #{templateName,jdbcType=VARCHAR}, - - - template_uuid = #{templateUuid,jdbcType=VARCHAR}, - - - key_id = #{keyId,jdbcType=BIGINT}, - - - config_value = #{configValue,jdbcType=VARCHAR}, - - - max_value = #{maxValue,jdbcType=VARCHAR}, - - - min_value = #{minValue,jdbcType=VARCHAR}, - - - validate_range = #{validateRange,jdbcType=VARCHAR}, - - - is_valid = #{isValid,jdbcType=VARCHAR}, - - - create_by = #{createBy,jdbcType=VARCHAR}, - - - update_by = #{updateBy,jdbcType=VARCHAR}, - - update_time = now(), - - where id = #{id,jdbcType=BIGINT} - - - update linkis_ps_configuration_template_config_key - set template_name = #{templateName,jdbcType=VARCHAR}, - template_uuid = #{templateUuid,jdbcType=VARCHAR}, - key_id = #{keyId,jdbcType=BIGINT}, - config_value = #{configValue,jdbcType=VARCHAR}, - max_value = #{maxValue,jdbcType=VARCHAR}, - min_value = #{minValue,jdbcType=VARCHAR}, - validate_range = #{validateRange,jdbcType=VARCHAR}, - is_valid = #{isValid,jdbcType=VARCHAR}, - create_by = #{createBy,jdbcType=VARCHAR}, - update_by = #{updateBy,jdbcType=VARCHAR}, - update_time = now() - where id = #{id,jdbcType=BIGINT} - - - + + + + diff --git a/linkis-public-enhancements/linkis-configuration/src/main/scala/org/apache/linkis/configuration/service/ConfigurationService.scala b/linkis-public-enhancements/linkis-configuration/src/main/scala/org/apache/linkis/configuration/service/ConfigurationService.scala index 6560af65ca..f47c2ca01e 100644 --- a/linkis-public-enhancements/linkis-configuration/src/main/scala/org/apache/linkis/configuration/service/ConfigurationService.scala +++ b/linkis-public-enhancements/linkis-configuration/src/main/scala/org/apache/linkis/configuration/service/ConfigurationService.scala @@ -17,9 +17,14 @@ package org.apache.linkis.configuration.service -import org.apache.linkis.common.utils.Logging +import org.apache.linkis.common.utils.{Logging, Utils} import org.apache.linkis.configuration.conf.Configuration -import org.apache.linkis.configuration.dao.{ConfigMapper, LabelMapper} +import org.apache.linkis.configuration.dao.{ + ConfigKeyLimitForUserMapper, + ConfigMapper, + LabelMapper, + TemplateConfigKeyMapper +} import org.apache.linkis.configuration.entity._ import org.apache.linkis.configuration.exception.ConfigurationException import org.apache.linkis.configuration.util.{LabelEntityParser, LabelParameterParser} @@ -42,6 +47,7 @@ import org.springframework.stereotype.Service import org.springframework.transaction.annotation.Transactional import org.springframework.util.CollectionUtils +import java.text.MessageFormat import java.util import scala.collection.JavaConverters._ @@ -57,6 +63,8 @@ class ConfigurationService extends Logging { @Autowired private var validatorManager: ValidatorManager = _ + @Autowired private var templateConfigKeyMapper: TemplateConfigKeyMapper = _ + private val combinedLabelBuilder: CombinedLabelBuilder = new CombinedLabelBuilder @Transactional @@ -172,6 +180,33 @@ class ConfigurationService extends Logging { createList: util.List[ConfigValue], updateList: util.List[ConfigValue] ): Any = { + + val configLabel = labelMapper.getLabelById(setting.getConfigLabelId) + val combinedLabel = combinedLabelBuilder + .buildFromStringValue(configLabel.getLabelKey, configLabel.getStringValue) + .asInstanceOf[CombinedLabel] + val templateConfigKeyVo = + templateConfigKeyMapper.selectByLabelAndKeyId(combinedLabel.getStringValue, setting.getId) + if (templateConfigKeyVo != null && StringUtils.isNotBlank(templateConfigKeyVo.getMaxValue)) { + Utils.tryCatch { + val maxValue = Integer.valueOf(templateConfigKeyVo.getMaxValue.replaceAll("[^0-9]", "")) + val configValue = Integer.valueOf(setting.getConfigValue.replaceAll("[^0-9]", "")) + if (configValue > maxValue) { + throw new ConfigurationException( + s"Parameter key:${setting.getKey},config value:${setting.getConfigValue} verification failed，exceeds the specified max value:${templateConfigKeyVo.getMaxValue}:(参数校验失败，超过指定的最大值):" + + s"${setting.getValidateType}--${setting.getValidateRange}" + ) + } + } { case exception: Exception => + if (exception.isInstanceOf[ConfigurationException]) { + throw exception + } else { + logger.warn( + s"Failed to check special limit setting for key:${setting.getKey},config value:${setting.getConfigValue}" + ) + } + } + } paramCheck(setting) if (setting.getIsUserDefined) { val configValue = new ConfigValue @@ -379,6 +414,32 @@ class ConfigurationService extends Logging { replaceCreatorToEngine(defaultCreatorConfigs, defaultEngineConfigs) } } + + // add special config limit info + if (defaultEngineConfigs.size() > 0) { + val keyIdList = defaultEngineConfigs.asScala.toStream + .map(e => { + e.getId + }) + .toList + .asJava + val limitList = + templateConfigKeyMapper.selectByLabelAndKeyIds(combinedLabel.getStringValue, keyIdList) + defaultEngineConfigs.asScala.foreach(entity => { + val keyId = entity.getId + val res = limitList.asScala.filter(v => v.getKeyId == keyId).toList.asJava + if (res.size() > 0) { + val specialMap = new util.HashMap[String, String]() + specialMap.put("maxValue", res.get(0).getMaxValue) + entity.setSpecialLimit(specialMap) + } + }) + } else { + logger.warn( + s"The configuration is empty. Please check the configuration information in the database table(配置为空,请检查数据库表中关于标签${combinedLabel.getStringValue}的配置信息是否完整)" + ) + } + (configs, defaultEngineConfigs) } From 6eb811297671b06242d014b3b08eb1259c77cbcb Mon Sep 17 00:00:00 2001 From: peacewong Date: Tue, 4 Jul 2023 21:20:29 +0800 Subject: [PATCH 189/261] Fix scala dont exit issue --- .../org/apache/linkis/governance/common/paser/CodeParser.scala | 3 +++ .../engineplugin/spark/executor/SparkScalaExecutor.scala | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/linkis-computation-governance/linkis-computation-governance-common/src/main/scala/org/apache/linkis/governance/common/paser/CodeParser.scala b/linkis-computation-governance/linkis-computation-governance-common/src/main/scala/org/apache/linkis/governance/common/paser/CodeParser.scala index aa99e7efd1..64ece62fd7 100644 --- a/linkis-computation-governance/linkis-computation-governance-common/src/main/scala/org/apache/linkis/governance/common/paser/CodeParser.scala +++ b/linkis-computation-governance/linkis-computation-governance-common/src/main/scala/org/apache/linkis/governance/common/paser/CodeParser.scala @@ -114,6 +114,9 @@ class ScalaCodeParser extends SingleCodeParser with Logging { case _ => } if (statementBuffer.nonEmpty) codeBuffer.append(statementBuffer.mkString("\n")) + // Make sure the last line is not a comment + codeBuffer.append("\n") + codeBuffer.append("val linkisVar=123") codeBuffer.toArray } diff --git a/linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/executor/SparkScalaExecutor.scala b/linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/executor/SparkScalaExecutor.scala index ebb2cdf4c9..6635e149e8 100644 --- a/linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/executor/SparkScalaExecutor.scala +++ b/linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/executor/SparkScalaExecutor.scala @@ -105,7 +105,7 @@ class SparkScalaExecutor(sparkEngineSession: SparkEngineSession, id: Long) System.setProperty("scala.repl.name.line", ("$line" + this.hashCode).replace('-', '0')) - setCodeParser(new EmptyCodeParser) + setCodeParser(new ScalaCodeParser) if (sparkILoop == null) { synchronized { From f060aebf5be1105e5891094e33e2ab54e7e7226a Mon Sep 17 00:00:00 2001 From: casionone Date: Thu, 6 Jul 2023 00:43:49 +0800 Subject: [PATCH 190/261] openEngineLog check with permission --- .../entrance/restful/EntranceRestfulApi.java | 5 +++-- .../manager/am/restful/EMRestfulApi.java | 21 ++++++++++++++++--- .../linkis/manager/rm/utils/RMUtilsTest.scala | 2 +- .../handler/CustomErrorAttributes.java | 6 +++++- 4 files changed, 27 insertions(+), 7 deletions(-) diff --git a/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/restful/EntranceRestfulApi.java b/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/restful/EntranceRestfulApi.java index 31a87f2f7c..f782ef2c6f 100644 --- a/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/restful/EntranceRestfulApi.java +++ b/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/restful/EntranceRestfulApi.java @@ -213,7 +213,7 @@ public Message status( message = Message.ok(); message.setMethod("/api/entrance/" + id + "/status"); message.data("status", job.get().getState().toString()).data("execID", id); - if(job.get().getErrorResponse()!=null){ + if (job.get().getErrorResponse() != null) { message.setMessage(job.get().getErrorResponse().message()); } } else { @@ -663,7 +663,8 @@ public Message kill( logger.error("kill job {} failed ", job.get().getId(), t); message = Message.error( - "An exception occurred while killing the job, kill failed(kill job的时候出现了异常，kill失败) with error:"+t.getMessage(), + "An exception occurred while killing the job, kill failed(kill job的时候出现了异常，kill失败) with error:" + + t.getMessage(), t); message.setMethod("/api/entrance/" + id + "/kill"); message.setStatus(1); diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/restful/EMRestfulApi.java b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/restful/EMRestfulApi.java index 05cda79c57..ccd691de0c 100644 --- a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/restful/EMRestfulApi.java +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/restful/EMRestfulApi.java @@ -430,11 +430,11 @@ public Message openEngineLog(HttpServletRequest req, @RequestBody JsonNode jsonN private Message executeECMOperation( EMNode ecmNode, String engineInstance, ECMOperateRequest ecmOperateRequest) { String operationName = OperateRequest$.MODULE$.getOperationName(ecmOperateRequest.parameters()); - if (ArrayUtils.contains(adminOperations, operationName) - && Configuration.isNotAdmin(ecmOperateRequest.user())) { + String userName = ecmOperateRequest.user(); + if (ArrayUtils.contains(adminOperations, operationName) && Configuration.isNotAdmin(userName)) { logger.warn( "User {} has no permission to execute {} admin Operation in ECM {}.", - ecmOperateRequest.user(), + userName, operationName, ecmNode.getServiceInstance()); return Message.error( @@ -455,6 +455,21 @@ private Message executeECMOperation( ecmOperateRequest.parameters().put("logDirSuffix", ecResourceInfoRecord.getLogDirSuffix()); } + // eg logDirSuffix -> root/20230705/io_file/6d48068a-0e1e-44b5-8eb2-835034db5b30/logs + String logDirSuffix = ecmOperateRequest.parameters().get("logDirSuffix").toString(); + String dirsuffix = logDirSuffix.split("/")[0]; + if (!dirsuffix.equals(userName) && Configuration.isNotAdmin(userName)) { + logger.warn( + "User {} has no permission to get log with path: {} in ECM:{}.", + userName, + logDirSuffix, + ecmNode.getServiceInstance()); + return Message.error( + "You have no permission to get log with path:" + + logDirSuffix + + " in ECM:" + + ecmNode.getServiceInstance()); + } ECMOperateResponse engineOperateResponse = ecmOperateService.executeOperation(ecmNode, ecmOperateRequest); diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/test/scala/org/apache/linkis/manager/rm/utils/RMUtilsTest.scala b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/test/scala/org/apache/linkis/manager/rm/utils/RMUtilsTest.scala index b95cd7d4f6..35039a394f 100644 --- a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/test/scala/org/apache/linkis/manager/rm/utils/RMUtilsTest.scala +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/test/scala/org/apache/linkis/manager/rm/utils/RMUtilsTest.scala @@ -35,7 +35,7 @@ class RMUtilsTest { maxResource ) assert( - " user Memory, requestResource : 600GB > availableResource : -2GB, maxResource : 19050GB." + " use Memory, requestResource : 600GB > availableResource : -2GB, maxResource : 19050GB." .equals(result) ) } diff --git a/linkis-spring-cloud-services/linkis-service-gateway/linkis-spring-cloud-gateway/src/main/java/org/apache/linkis/gateway/springcloud/handler/CustomErrorAttributes.java b/linkis-spring-cloud-services/linkis-service-gateway/linkis-spring-cloud-gateway/src/main/java/org/apache/linkis/gateway/springcloud/handler/CustomErrorAttributes.java index 4cbafa6625..ff27b76b47 100644 --- a/linkis-spring-cloud-services/linkis-service-gateway/linkis-spring-cloud-gateway/src/main/java/org/apache/linkis/gateway/springcloud/handler/CustomErrorAttributes.java +++ b/linkis-spring-cloud-services/linkis-service-gateway/linkis-spring-cloud-gateway/src/main/java/org/apache/linkis/gateway/springcloud/handler/CustomErrorAttributes.java @@ -47,7 +47,11 @@ public Map getErrorAttributes( Map map = new HashMap<>(); map.put("method", request.path()); map.put("status", errorStatus.value()); - map.put("message", errorStatus.getReasonPhrase()); + String msg = errorStatus.getReasonPhrase(); + if (errorStatus.value() >= HttpStatus.INTERNAL_SERVER_ERROR.value()) { + msg = msg + ", with request path:" + request.path(); + } + map.put("message", msg); map.put("data", Lists.newArrayList()); return map; From 89b85ff5251af3693b121534bd1482cc4feb89e1 Mon Sep 17 00:00:00 2001 From: v_nikeyang <1013195908@qq.com> Date: Thu, 6 Jul 2023 18:24:19 +0800 Subject: [PATCH 191/261] flink-log4j --- .../launch/FlinkEngineConnLaunchBuilder.scala | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/linkis-engineconn-plugins/flink/src/main/scala/org/apache/linkis/engineconnplugin/flink/launch/FlinkEngineConnLaunchBuilder.scala b/linkis-engineconn-plugins/flink/src/main/scala/org/apache/linkis/engineconnplugin/flink/launch/FlinkEngineConnLaunchBuilder.scala index cbcae4ea54..2452799ced 100644 --- a/linkis-engineconn-plugins/flink/src/main/scala/org/apache/linkis/engineconnplugin/flink/launch/FlinkEngineConnLaunchBuilder.scala +++ b/linkis-engineconn-plugins/flink/src/main/scala/org/apache/linkis/engineconnplugin/flink/launch/FlinkEngineConnLaunchBuilder.scala @@ -24,12 +24,12 @@ import org.apache.linkis.hadoop.common.conf.HadoopConf import org.apache.linkis.manager.common.protocol.bml.BmlResource import org.apache.linkis.manager.engineplugin.common.conf.EnvConfiguration import org.apache.linkis.manager.engineplugin.common.launch.entity.EngineConnBuildRequest -import org.apache.linkis.manager.engineplugin.common.launch.process.Environment.{variable, USER} -import org.apache.linkis.manager.engineplugin.common.launch.process.JavaProcessEngineConnLaunchBuilder +import org.apache.linkis.manager.engineplugin.common.launch.process.Environment.{USER,PWD, variable} +import org.apache.linkis.manager.engineplugin.common.launch.process.{Environment, JavaProcessEngineConnLaunchBuilder} +import org.apache.linkis.manager.engineplugin.common.launch.process.LaunchConstants.{CLASS_PATH_SEPARATOR, addPathToClassPath} import org.apache.linkis.manager.label.entity.engine.UserCreatorLabel import java.util - import scala.collection.JavaConverters._ class FlinkEngineConnLaunchBuilder extends JavaProcessEngineConnLaunchBuilder { @@ -81,6 +81,18 @@ class FlinkEngineConnLaunchBuilder extends JavaProcessEngineConnLaunchBuilder { } bmlResources } + override def getEnvironment(implicit + engineConnBuildRequest: EngineConnBuildRequest + ): util.Map[String, String] = { + val environment = new util.HashMap[String, String] + addPathToClassPath(environment, variable(PWD)) + val linkisEnvironment = super.getEnvironment + val linkisClassPath = linkisEnvironment.get(Environment.CLASSPATH.toString) + val v = environment.get(Environment.CLASSPATH.toString) + CLASS_PATH_SEPARATOR + linkisClassPath + environment.put(Environment.CLASSPATH.toString, v) + logger.info(environment.asScala.map(e => s"${e._1}->${e._2}").mkString(",")) + environment + } private def contentToBmlResource(userName: String, content: String): BmlResource = { val contentMap = JsonUtils.jackson.readValue(content, classOf[util.Map[String, Object]]) From 308ceb832a7eb5bede4f22d2e78b621c3f100018 Mon Sep 17 00:00:00 2001 From: v-kkhuang <62878639+v-kkhuang@users.noreply.github.com> Date: Fri, 7 Jul 2023 12:11:36 +0800 Subject: [PATCH 192/261] =?UTF-8?q?=E3=80=901.1.13=E3=80=91=20Operation=20?= =?UTF-8?q?and=20maintenance=20optimization:=20Service=20version=20informa?= =?UTF-8?q?tion=20reporting=20(#203)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * code optimize for load version.properties --------- Co-authored-by: huangKai-2323 <62878639+huangKai-2323@users.noreply.github.com> Co-authored-by: casionone --- .../scala/org/apache/linkis/common/conf/BDPConfiguration.scala | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/linkis-commons/linkis-common/src/main/scala/org/apache/linkis/common/conf/BDPConfiguration.scala b/linkis-commons/linkis-common/src/main/scala/org/apache/linkis/common/conf/BDPConfiguration.scala index 214beb881b..9ea76fa3aa 100644 --- a/linkis-commons/linkis-common/src/main/scala/org/apache/linkis/common/conf/BDPConfiguration.scala +++ b/linkis-commons/linkis-common/src/main/scala/org/apache/linkis/common/conf/BDPConfiguration.scala @@ -102,9 +102,8 @@ private[conf] object BDPConfiguration extends Logging { } // load version conf val versionConf = sysProps.getOrElse("linkis.version.conf", DEFAULT_VERSION_FILE_NAME) - // version conf file path(env LINKIS_VERSION_CONF_FILE_PATH > classpath) - var versionConfPath = sysProps.getOrElse("LINKIS_VERSION_CONF_FILE_PATH", "") + var versionConfPath = System.getenv("LINKIS_VERSION_CONF_FILE_PATH") + "/" + versionConf if (StringUtils.isBlank(versionConfPath)) { logger.info( s"LINKIS_VERSION_CONF_FILE_PATH is empty, try to use version.properties file path from classpath" From 3b1206812a68718457cd5082306a40ea738af27b Mon Sep 17 00:00:00 2001 From: v-kkhuang <62878639+v-kkhuang@users.noreply.github.com> Date: Fri, 7 Jul 2023 12:12:52 +0800 Subject: [PATCH 193/261] =?UTF-8?q?=E3=80=901.1.13=E3=80=91Adding=20specif?= =?UTF-8?q?ied=20parameter=20validation=20when=20modifying=20configuration?= =?UTF-8?q?=20files=20(#201)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * config add value intercept --------- Co-authored-by: huangKai-2323 <62878639+huangKai-2323@users.noreply.github.com> Co-authored-by: casionone --- .../restful/api/ConfigurationRestfulApi.java | 13 ++++- .../service/impl/TenantConfigServiceImpl.java | 56 +++++++++---------- 2 files changed, 40 insertions(+), 29 deletions(-) diff --git a/linkis-public-enhancements/linkis-configuration/src/main/java/org/apache/linkis/configuration/restful/api/ConfigurationRestfulApi.java b/linkis-public-enhancements/linkis-configuration/src/main/java/org/apache/linkis/configuration/restful/api/ConfigurationRestfulApi.java index 83aeb1e367..1cb874aec7 100644 --- a/linkis-public-enhancements/linkis-configuration/src/main/java/org/apache/linkis/configuration/restful/api/ConfigurationRestfulApi.java +++ b/linkis-public-enhancements/linkis-configuration/src/main/java/org/apache/linkis/configuration/restful/api/ConfigurationRestfulApi.java @@ -454,12 +454,14 @@ public Message getKeyValue( @RequestMapping(path = "/keyvalue", method = RequestMethod.POST) public Message saveKeyValue(HttpServletRequest req, @RequestBody Map json) throws ConfigurationException { + Message message = Message.ok(); String username = ModuleUserUtils.getOperationUser(req, "saveKey"); String engineType = (String) json.getOrDefault("engineType", "*"); String version = (String) json.getOrDefault("version", "*"); String creator = (String) json.getOrDefault("creator", "*"); String configKey = (String) json.get("configKey"); String value = (String) json.get("configValue"); + boolean force = Boolean.parseBoolean(json.getOrDefault("force", "false").toString()); if (engineType.equals("*") && !version.equals("*")) { return Message.error("When engineType is any engine, the version must also be any version"); } @@ -474,9 +476,18 @@ public Message saveKeyValue(HttpServletRequest req, @RequestBody Map tenantResult = new AtomicReference<>(false); + // Obtain the tenant information of the ECM list + Map ecmListResult = null; + try { + ecmListResult = HttpsUtil.sendHttp(null, null); + logger.info("Request ecm list response {}:", ecmListResult); + } catch (IOException e) { + logger.warn("failed to get ecmResource data"); + } + Map>> data = MapUtils.getMap(ecmListResult, "data"); + List> emNodeVoList = data.get("EMs"); + // Compare ECM list tenant labels for task + emNodeVoList.forEach( + ecmInfo -> { + List> labels = (List>) ecmInfo.get("labels"); + labels.stream() + .filter(labelmap -> labelmap.containsKey("tenant")) + .forEach( + map -> { + String tenant = map.get("tenant").toString().toLowerCase(); + if (tenant.equals(tenantVo.getTenantValue().toLowerCase())) { + tenantResult.set(true); + } + }); + }); + // Compare the value of ecm tenant + if (!tenantResult.get()) + throw new ConfigurationException("The ECM with the corresponding label was not found"); if (!tenantVo.getCreator().equals("*")) { - AtomicReference tenantResult = new AtomicReference<>(false); - // Obtain the tenant information of the ECM list - Map ecmListResult = null; - try { - ecmListResult = HttpsUtil.sendHttp(null, null); - logger.info("Request ecm list response {}:", ecmListResult); - } catch (IOException e) { - logger.warn("failed to get ecmResource data"); - } - Map>> data = MapUtils.getMap(ecmListResult, "data"); - List> emNodeVoList = data.get("EMs"); - // Compare ECM list tenant labels for task - emNodeVoList.forEach( - ecmInfo -> { - List> labels = (List>) ecmInfo.get("labels"); - labels.stream() - .filter(labelmap -> labelmap.containsKey("tenant")) - .forEach( - map -> { - String tenant = map.get("tenant").toString().toLowerCase(); - if (tenant.equals(tenantVo.getTenantValue().toLowerCase())) { - tenantResult.set(true); - } - }); - }); - // Compare the value of ecm tenant - if (!tenantResult.get()) - throw new ConfigurationException("The ECM with the corresponding label was not found"); // The beginning of tenantValue needs to contain creator String creator = tenantVo.getCreator().toLowerCase(); String[] tenantArray = tenantVo.getTenantValue().toLowerCase().split("_"); From 50bb11d946b9f308ff4044c4562ae905159e0945 Mon Sep 17 00:00:00 2001 From: casionone Date: Fri, 7 Jul 2023 12:24:51 +0800 Subject: [PATCH 194/261] openEngineLog check with historyAdmin permission configuration template optimize --- .../am/restful/ECResourceInfoRestfulApi.java | 2 +- .../linkis/manager/am/restful/EMRestfulApi.java | 2 +- .../restful/api/ConfigurationRestfulApi.java | 9 +++++++++ .../impl/TemplateConfigKeyServiceImpl.java | 17 ++++++++++++++--- 4 files changed, 25 insertions(+), 5 deletions(-) diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/restful/ECResourceInfoRestfulApi.java b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/restful/ECResourceInfoRestfulApi.java index df152b7008..5c5d625918 100644 --- a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/restful/ECResourceInfoRestfulApi.java +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/restful/ECResourceInfoRestfulApi.java @@ -148,7 +148,7 @@ public Message queryEcrHistory( calendar.set(Calendar.SECOND, 0); startDate = calendar.getTime(); } - if (Configuration.isAdmin(username)) { + if (Configuration.isJobHistoryAdmin(username)) { username = null; if (StringUtils.isNotBlank(creatorUser)) { username = creatorUser; diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/restful/EMRestfulApi.java b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/restful/EMRestfulApi.java index ccd691de0c..f8a3ce76ae 100644 --- a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/restful/EMRestfulApi.java +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/restful/EMRestfulApi.java @@ -458,7 +458,7 @@ private Message executeECMOperation( // eg logDirSuffix -> root/20230705/io_file/6d48068a-0e1e-44b5-8eb2-835034db5b30/logs String logDirSuffix = ecmOperateRequest.parameters().get("logDirSuffix").toString(); String dirsuffix = logDirSuffix.split("/")[0]; - if (!dirsuffix.equals(userName) && Configuration.isNotAdmin(userName)) { + if (!dirsuffix.equals(userName) && Configuration.isNotJobHistoryAdmin(userName)) { logger.warn( "User {} has no permission to get log with path: {} in ECM:{}.", userName, diff --git a/linkis-public-enhancements/linkis-configuration/src/main/java/org/apache/linkis/configuration/restful/api/ConfigurationRestfulApi.java b/linkis-public-enhancements/linkis-configuration/src/main/java/org/apache/linkis/configuration/restful/api/ConfigurationRestfulApi.java index 1cb874aec7..d7487f7461 100644 --- a/linkis-public-enhancements/linkis-configuration/src/main/java/org/apache/linkis/configuration/restful/api/ConfigurationRestfulApi.java +++ b/linkis-public-enhancements/linkis-configuration/src/main/java/org/apache/linkis/configuration/restful/api/ConfigurationRestfulApi.java @@ -179,6 +179,15 @@ public Message getItemList( temp.put("validateType", configKey.getValidateType()); temp.put("validateRange", configKey.getValidateRange()); temp.put("boundaryType", configKey.getBoundaryType()); + temp.put("defaultValue", configKey.getDefaultValue()); + temp.put("description", configKey.getDescription()); + // for front-end to judge whether input is required + if (StringUtils.isNotEmpty(configKey.getDefaultValue())) { + temp.put("require", "true"); + } else { + temp.put("require", "false"); + } + filterResult.add(temp); } diff --git a/linkis-public-enhancements/linkis-configuration/src/main/java/org/apache/linkis/configuration/service/impl/TemplateConfigKeyServiceImpl.java b/linkis-public-enhancements/linkis-configuration/src/main/java/org/apache/linkis/configuration/service/impl/TemplateConfigKeyServiceImpl.java index 467b846258..b990c9ee8e 100644 --- a/linkis-public-enhancements/linkis-configuration/src/main/java/org/apache/linkis/configuration/service/impl/TemplateConfigKeyServiceImpl.java +++ b/linkis-public-enhancements/linkis-configuration/src/main/java/org/apache/linkis/configuration/service/impl/TemplateConfigKeyServiceImpl.java @@ -105,9 +105,10 @@ public Boolean updateKeyMapping( String configValue = item.getConfigValue(); String maxValue = item.getMaxValue(); - if (!validatorManager - .getOrCreateValidator(validateType) - .validate(configValue, validateRange)) { + if (StringUtils.isNotEmpty(configValue) + && !validatorManager + .getOrCreateValidator(validateType) + .validate(configValue, validateRange)) { String msg = MessageFormat.format( "Parameter configValue verification failed(参数configValue校验失败):" @@ -258,6 +259,16 @@ public Map apply( templateUid); throw new ConfigurationException(msg); } + // check input engineType is same as template key engineType + ConfigKey configKey = configMapper.selectKeyByKeyID(templateConfigKeyList.get(0).getKeyId()); + if (!engineType.equals(configKey.getEngineType())) { + String msg = + MessageFormat.format( + "The engineType:{0} associated with the template:{1} does not match the input engineType:{2}, please check whether the parameters are correct" + + "(模板关联的引擎类型：{0} 和下发的引擎类型：{2} 不匹配,请检查参数是否正确)", + configKey.getEngineType(), templateUid, engineType); + throw new ConfigurationException(msg); + } for (String user : userList) { // try to create combined_userCreator_engineType label for user Map res = new HashMap(); From 29c78f655758bf5424e2a5ef0eefc6ed98e95129 Mon Sep 17 00:00:00 2001 From: casionone Date: Fri, 7 Jul 2023 12:26:48 +0800 Subject: [PATCH 195/261] revert sql result deal with \n --- .../linkis/engineplugin/spark/executor/SQLSession.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/executor/SQLSession.scala b/linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/executor/SQLSession.scala index 2c70836a69..ba0af6b64b 100644 --- a/linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/executor/SQLSession.scala +++ b/linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/executor/SQLSession.scala @@ -188,7 +188,7 @@ object SQLSession extends Logging { .sorted .mkString("{", ",", "}") case (null, _) => "null" - // case (str: String, StringType) => str.replaceAll("\n|\t", " ") + case (str: String, StringType) => str.replaceAll("\n|\t", " ") case (double: Double, DoubleType) => nf.format(double) case (decimal: java.math.BigDecimal, DecimalType()) => formatDecimal(decimal) case (other: Any, tpe) => other.toString @@ -213,7 +213,7 @@ object SQLSession extends Logging { .sorted .mkString("{", ",", "}") - // case (str: String, StringType) => str.replaceAll("\n|\t", " ") + case (str: String, StringType) => str.replaceAll("\n|\t", " ") case (double: Double, DoubleType) => if (double.isNaN) { "NaN" From 6dfd4b441bf367d690fde6838113732761e8a448 Mon Sep 17 00:00:00 2001 From: ahaoyao <731753729@qq.com> Date: Fri, 7 Jul 2023 19:09:57 +0800 Subject: [PATCH 196/261] rollback --- .../org/apache/linkis/entrance/restful/EntranceRestfulApi.java | 3 --- 1 file changed, 3 deletions(-) diff --git a/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/restful/EntranceRestfulApi.java b/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/restful/EntranceRestfulApi.java index f782ef2c6f..b616c9c658 100644 --- a/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/restful/EntranceRestfulApi.java +++ b/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/restful/EntranceRestfulApi.java @@ -213,9 +213,6 @@ public Message status( message = Message.ok(); message.setMethod("/api/entrance/" + id + "/status"); message.data("status", job.get().getState().toString()).data("execID", id); - if (job.get().getErrorResponse() != null) { - message.setMessage(job.get().getErrorResponse().message()); - } } else { message = Message.error( From dbcc74c1e770b265c81f1d699be747609e75aa87 Mon Sep 17 00:00:00 2001 From: huangKai-2323 <62878639+huangKai-2323@users.noreply.github.com> Date: Thu, 6 Jul 2023 14:18:21 +0800 Subject: [PATCH 197/261] code optimization --- linkis-dist/package/db/linkis_ddl.sql | 2 +- .../package/db/upgrade/1.4.1_schema/mysql/linkis_ddl.sql | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/linkis-dist/package/db/linkis_ddl.sql b/linkis-dist/package/db/linkis_ddl.sql index c62ee4b76f..47ab5d74a8 100644 --- a/linkis-dist/package/db/linkis_ddl.sql +++ b/linkis-dist/package/db/linkis_ddl.sql @@ -497,6 +497,7 @@ CREATE TABLE if not exists `linkis_ps_bml_resources` ( `update_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'Updated time', `updator` varchar(50) DEFAULT NULL COMMENT 'updator', `enable_flag` tinyint(1) NOT NULL DEFAULT '1' COMMENT 'Status, 1: normal, 0: frozen', + unique key `uniq_rid_eflag`(`resource_id`, `enable_flag`), PRIMARY KEY (`id`) ) ENGINE=InnoDB AUTO_INCREMENT=9 DEFAULT CHARSET=utf8mb4; @@ -518,7 +519,6 @@ CREATE TABLE if not exists `linkis_ps_bml_resources_version` ( `updator` varchar(50) DEFAULT NULL COMMENT 'updator', `enable_flag` tinyint(1) NOT NULL DEFAULT '1' COMMENT 'Status, 1: normal, 0: frozen', unique key `uniq_rid_version`(`resource_id`, `version`), - unique key `uniq_rid_eflag`(`resource_id`, `enable_flag`), PRIMARY KEY (`id`) ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; diff --git a/linkis-dist/package/db/upgrade/1.4.1_schema/mysql/linkis_ddl.sql b/linkis-dist/package/db/upgrade/1.4.1_schema/mysql/linkis_ddl.sql index 0abb1ed5b4..2e1463ca55 100644 --- a/linkis-dist/package/db/upgrade/1.4.1_schema/mysql/linkis_ddl.sql +++ b/linkis-dist/package/db/upgrade/1.4.1_schema/mysql/linkis_ddl.sql @@ -18,5 +18,5 @@ ALTER TABLE `linkis_ps_udf_user_load` ADD CONSTRAINT `uniq_uid_uname` UNIQUE (`udf_id`, `user_name`); -ALTER TABLE `linkis_ps_bml_resources_version` ADD CONSTRAINT `uniq_rid_eflag` UNIQUE (`resource_id`, `enable_flag`); +ALTER TABLE `linkis_ps_bml_resources` ADD CONSTRAINT `uniq_rid_eflag` UNIQUE (`resource_id`, `enable_flag`); From 147c7101134a3834e0d0b2cdba92e7d241867f28 Mon Sep 17 00:00:00 2001 From: v-kkhuang <62878639+v-kkhuang@users.noreply.github.com> Date: Mon, 10 Jul 2023 19:31:13 +0800 Subject: [PATCH 198/261] code optimize after code review (#209) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit code revenrt： 1.Version information messages are processed through script 2.The fileinfo interface returns totalline code review ： 1.The exception of the configuration parameter check is specific to the configuration exception add configration exception catch --------- Co-authored-by: huangKai-2323 <62878639+huangKai-2323@users.noreply.github.com> --- .../linkis/common/conf/BDPConfiguration.scala | 26 ------------------- .../server/utils/LinkisMainHelper.scala | 4 +-- .../storage/utils/FileSystemUtils.scala | 19 -------------- .../restful/api/ConfigurationRestfulApi.java | 2 +- .../filesystem/restful/api/FsRestfulApi.java | 8 ------ 5 files changed, 2 insertions(+), 57 deletions(-) diff --git a/linkis-commons/linkis-common/src/main/scala/org/apache/linkis/common/conf/BDPConfiguration.scala b/linkis-commons/linkis-common/src/main/scala/org/apache/linkis/common/conf/BDPConfiguration.scala index 9ea76fa3aa..14febab63a 100644 --- a/linkis-commons/linkis-common/src/main/scala/org/apache/linkis/common/conf/BDPConfiguration.scala +++ b/linkis-commons/linkis-common/src/main/scala/org/apache/linkis/common/conf/BDPConfiguration.scala @@ -36,8 +36,6 @@ private[conf] object BDPConfiguration extends Logging { val DEFAULT_SERVER_CONF_FILE_NAME = "linkis-server.properties" - val DEFAULT_VERSION_FILE_NAME = "version.properties" - val DEFAULT_CONFIG_HOT_LOAD_DELAY_MILLS = 3 * 60 * 1000L private val extractConfig = new Properties @@ -100,31 +98,7 @@ private[conf] object BDPConfiguration extends Logging { } } } - // load version conf - val versionConf = sysProps.getOrElse("linkis.version.conf", DEFAULT_VERSION_FILE_NAME) - // version conf file path(env LINKIS_VERSION_CONF_FILE_PATH > classpath) - var versionConfPath = System.getenv("LINKIS_VERSION_CONF_FILE_PATH") + "/" + versionConf - if (StringUtils.isBlank(versionConfPath)) { - logger.info( - s"LINKIS_VERSION_CONF_FILE_PATH is empty, try to use version.properties file path from classpath" - ) - val versionConfFileURL = getClass.getClassLoader.getResource(versionConf) - if (versionConfFileURL != null) { - versionConfPath = versionConfFileURL.getPath - } - } - if (new File(versionConfPath).exists) { - logger.info( - s"*********************** Notice: The Linkis version file is $versionConf ! ******************" - ) - initConfig(config, versionConfPath) - configList.append(versionConfPath) - } else { - logger.warn( - s"**************** Notice: The Linkis version file $versionConf does not exist! *******************" - ) - } // init hot-load config task val hotLoadTask = new Runnable { override def run(): Unit = { diff --git a/linkis-commons/linkis-module/src/main/scala/org/apache/linkis/server/utils/LinkisMainHelper.scala b/linkis-commons/linkis-module/src/main/scala/org/apache/linkis/server/utils/LinkisMainHelper.scala index d7d28361fe..7614bbef2a 100644 --- a/linkis-commons/linkis-module/src/main/scala/org/apache/linkis/server/utils/LinkisMainHelper.scala +++ b/linkis-commons/linkis-module/src/main/scala/org/apache/linkis/server/utils/LinkisMainHelper.scala @@ -61,9 +61,7 @@ object LinkisMainHelper { } resArr = resArr :+ s"--prometheus.endpoint=$prometheusEndpoint" } - val version = CommonVars("version", "").getValue - resArr = resArr :+ s"--eureka.instance.metadata-map.linkis.app.version=$version" - resArr + return resArr } } diff --git a/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/utils/FileSystemUtils.scala b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/utils/FileSystemUtils.scala index 0e3949b2b3..5252c12e03 100644 --- a/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/utils/FileSystemUtils.scala +++ b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/utils/FileSystemUtils.scala @@ -22,9 +22,6 @@ import org.apache.linkis.common.utils.{Logging, Utils} import org.apache.linkis.storage.FSFactory import org.apache.linkis.storage.fs.FileSystem import org.apache.linkis.storage.fs.impl.LocalFileSystem -import org.apache.linkis.storage.source.FileSource - -import org.apache.commons.io.IOUtils import java.io.IOException import java.util @@ -116,20 +113,4 @@ object FileSystemUtils extends Logging { true } - /** - * Return the total number of lines in the file - * - * @param fsPath - * @param fileSystem - * @return - * TotalLine - */ - def getTotalLine(fsPath: FsPath, fileSystem: FileSystem): Int = { - val fileSource = FileSource.create(fsPath, fileSystem) - try { - fileSource.collect() - fileSource.getTotalLine - } finally IOUtils.closeQuietly(fileSource) - } - } diff --git a/linkis-public-enhancements/linkis-configuration/src/main/java/org/apache/linkis/configuration/restful/api/ConfigurationRestfulApi.java b/linkis-public-enhancements/linkis-configuration/src/main/java/org/apache/linkis/configuration/restful/api/ConfigurationRestfulApi.java index d7487f7461..0632ca6aaa 100644 --- a/linkis-public-enhancements/linkis-configuration/src/main/java/org/apache/linkis/configuration/restful/api/ConfigurationRestfulApi.java +++ b/linkis-public-enhancements/linkis-configuration/src/main/java/org/apache/linkis/configuration/restful/api/ConfigurationRestfulApi.java @@ -488,7 +488,7 @@ public Message saveKeyValue(HttpServletRequest req, @RequestBody Map[] fileInfo = fileSource.getFileInfo(pageSize); IOUtils.closeQuietly(fileSource); if (null != fileInfo && fileInfo.length > 0) { - int rowNumber = (int) fileInfo[0].getSecond(); message.data("path", path); message.data("colNumber", fileInfo[0].getFirst()); message.data("rowNumber", fileInfo[0].getSecond()); - if (rowNumber >= pageSize) { - message.data("totalLine", FileSystemUtils.getTotalLine(fsPath, fileSystem)); - } else { - message.data("totalLine", rowNumber); - } } else { message.data("path", path); message.data("colNumber", 0); message.data("rowNumber", 0); - message.data("totalLine", 0); } return message; } finally { From bf25933cc1bba2396d44d67a54ca86480c6e8bef Mon Sep 17 00:00:00 2001 From: Casion Date: Tue, 11 Jul 2023 11:33:03 +0800 Subject: [PATCH 199/261] code optimize after code review (#206) * code optimize after code review 1 VariableOperation for json type 2 JobHistoryMapper slow sql optimize 3 TemplateConfigKeyServiceImpl apply add transaction for mapper * fix junit test * optmize slow query sql * variableutils support FlinkCodeOnceExcutor --- .../common/utils/VariableOperationUtils.java | 27 ++++++++++++++++ .../linkis/common/utils/VariableUtils.scala | 12 ++++++- .../common/paser/ScalaCodeParserTest.scala | 10 +++--- .../manager/am/restful/EMRestfulApi.java | 32 +++++++++---------- .../restful/api/ConfigurationRestfulApi.java | 1 - .../impl/TemplateConfigKeyServiceImpl.java | 30 ++++++++++++++--- .../cache/impl/DefaultQueryCacheManager.java | 7 +--- .../jobhistory/dao/JobHistoryMapper.java | 6 ++-- .../mapper/common/JobHistoryMapper.xml | 6 ++-- .../jobhistory/dao/JobHistoryMapperTest.java | 7 ++-- 10 files changed, 95 insertions(+), 43 deletions(-) diff --git a/linkis-commons/linkis-common/src/main/java/org/apache/linkis/common/utils/VariableOperationUtils.java b/linkis-commons/linkis-common/src/main/java/org/apache/linkis/common/utils/VariableOperationUtils.java index 55bae5b2eb..615472474d 100644 --- a/linkis-commons/linkis-common/src/main/java/org/apache/linkis/common/utils/VariableOperationUtils.java +++ b/linkis-commons/linkis-common/src/main/java/org/apache/linkis/common/utils/VariableOperationUtils.java @@ -84,6 +84,7 @@ public static ZonedDateTime toZonedDateTime(Date date) { * @param str * @return */ + @Deprecated public static String replaces(ZonedDateTime dateTime, String str) throws VariableOperationFailedException { try { @@ -98,6 +99,32 @@ public static String replaces(ZonedDateTime dateTime, String str) return replace(dateTime, str); } + /** + * json support variable operation + * + * @param codeType + * @param dateTime + * @param str + * @return + */ + public static String replaces(String codeType, ZonedDateTime dateTime, String str) + throws VariableOperationFailedException { + String languageType = CodeAndRunTypeUtils.getLanguageTypeByCodeType(codeType, ""); + if (languageType.equals(CodeAndRunTypeUtils.LANGUAGE_TYPE_JSON())) { + try { + JsonNode rootNode = mapper.readTree(str); + if (rootNode.isArray() || rootNode.isObject()) { + replaceJson(dateTime, rootNode); + return rootNode.toString(); + } + } catch (Exception e) { + return replace(dateTime, str); + } + } + + return replace(dateTime, str); + } + /** * @param dateTime * @param str diff --git a/linkis-commons/linkis-common/src/main/scala/org/apache/linkis/common/utils/VariableUtils.scala b/linkis-commons/linkis-common/src/main/scala/org/apache/linkis/common/utils/VariableUtils.scala index 1600b7eb18..6c5bd7cf3c 100644 --- a/linkis-commons/linkis-common/src/main/scala/org/apache/linkis/common/utils/VariableUtils.scala +++ b/linkis-commons/linkis-common/src/main/scala/org/apache/linkis/common/utils/VariableUtils.scala @@ -143,9 +143,10 @@ object VariableUtils extends Logging { } initAllDateVars(run_date, nameAndType) val codeOperation = parserVar(code, nameAndType) - parserDate(codeOperation, run_date) + parserDate(codeType, codeOperation, run_date) } + @deprecated private def parserDate(code: String, run_date: CustomDateType): String = { if (Configuration.VARIABLE_OPERATION) { val zonedDateTime: ZonedDateTime = VariableOperationUtils.toZonedDateTime(run_date.getDate) @@ -155,6 +156,15 @@ object VariableUtils extends Logging { } } + private def parserDate(codeType: String, code: String, run_date: CustomDateType): String = { + if (Configuration.VARIABLE_OPERATION) { + val zonedDateTime: ZonedDateTime = VariableOperationUtils.toZonedDateTime(run_date.getDate) + VariableOperationUtils.replaces(codeType, zonedDateTime, code) + } else { + code + } + } + private def initAllDateVars( run_date: CustomDateType, nameAndType: mutable.Map[String, variable.VariableType] diff --git a/linkis-computation-governance/linkis-computation-governance-common/src/test/scala/org/apache/linkis/governance/common/paser/ScalaCodeParserTest.scala b/linkis-computation-governance/linkis-computation-governance-common/src/test/scala/org/apache/linkis/governance/common/paser/ScalaCodeParserTest.scala index 3cfe787f61..04adf3446c 100644 --- a/linkis-computation-governance/linkis-computation-governance-common/src/test/scala/org/apache/linkis/governance/common/paser/ScalaCodeParserTest.scala +++ b/linkis-computation-governance/linkis-computation-governance-common/src/test/scala/org/apache/linkis/governance/common/paser/ScalaCodeParserTest.scala @@ -29,7 +29,7 @@ class ScalaCodeParserTest { "val codeBuffer = new ArrayBuffer[String]()\n val statementBuffer = new ArrayBuffer[String]()" val scalaCodeParser = new ScalaCodeParser val array = scalaCodeParser.parse(scalaCode) - Assertions.assertTrue(array.size == 1) + Assertions.assertTrue(array.size == 3) } @@ -41,7 +41,7 @@ class ScalaCodeParserTest { " def addInt( a:Int, b:Int )\n var sum:Int = 0\n sum = a + b\n return sum\n }" val scalaCodeParser = new ScalaCodeParser val array = scalaCodeParser.parse(abnormalCode) - Assertions.assertTrue(array.length == 1) + Assertions.assertTrue(array.length == 3) } @@ -54,7 +54,7 @@ class ScalaCodeParserTest { val scalaCodeParser = new ScalaCodeParser val array = scalaCodeParser.parse(importCode) - Assertions.assertTrue(array.length == 2) + Assertions.assertTrue(array.length == 4) } @@ -68,7 +68,7 @@ class ScalaCodeParserTest { val scalaCodeParser = new ScalaCodeParser val arrayResult1 = scalaCodeParser.parse(specialCodeExp1) - Assertions.assertTrue(arrayResult1.length == 2) + Assertions.assertTrue(arrayResult1.length == 4) val specialCodeExp2 = " @BeanProperty\n var id: Long = _\n @BeanProperty\n var status: Int = 0\n " + @@ -79,7 +79,7 @@ class ScalaCodeParserTest { ".append(data, that.data)\n .isEquals\n }" val arrayResult2 = scalaCodeParser.parse(specialCodeExp2) - Assertions.assertTrue(arrayResult2.length == 1) + Assertions.assertTrue(arrayResult2.length == 3) } diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/restful/EMRestfulApi.java b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/restful/EMRestfulApi.java index f8a3ce76ae..8218896c8a 100644 --- a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/restful/EMRestfulApi.java +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/restful/EMRestfulApi.java @@ -452,24 +452,24 @@ private Message executeECMOperation( if (Objects.isNull(ecResourceInfoRecord)) { return Message.error("ECM instance: " + ecmNode.getServiceInstance() + " not exist "); } - ecmOperateRequest.parameters().put("logDirSuffix", ecResourceInfoRecord.getLogDirSuffix()); + // eg logDirSuffix -> root/20230705/io_file/6d48068a-0e1e-44b5-8eb2-835034db5b30/logs + String logDirSuffix = ecResourceInfoRecord.getLogDirSuffix(); + if (!userName.equals(ecResourceInfoRecord.getCreateUser()) + && Configuration.isNotJobHistoryAdmin(userName)) { + logger.warn( + "User {} has no permission to get log with path: {} in ECM:{}.", + userName, + logDirSuffix, + ecmNode.getServiceInstance()); + return Message.error( + "You have no permission to get log with path:" + + logDirSuffix + + " in ECM:" + + ecmNode.getServiceInstance()); + } + ecmOperateRequest.parameters().put("logDirSuffix", logDirSuffix); } - // eg logDirSuffix -> root/20230705/io_file/6d48068a-0e1e-44b5-8eb2-835034db5b30/logs - String logDirSuffix = ecmOperateRequest.parameters().get("logDirSuffix").toString(); - String dirsuffix = logDirSuffix.split("/")[0]; - if (!dirsuffix.equals(userName) && Configuration.isNotJobHistoryAdmin(userName)) { - logger.warn( - "User {} has no permission to get log with path: {} in ECM:{}.", - userName, - logDirSuffix, - ecmNode.getServiceInstance()); - return Message.error( - "You have no permission to get log with path:" - + logDirSuffix - + " in ECM:" - + ecmNode.getServiceInstance()); - } ECMOperateResponse engineOperateResponse = ecmOperateService.executeOperation(ecmNode, ecmOperateRequest); diff --git a/linkis-public-enhancements/linkis-configuration/src/main/java/org/apache/linkis/configuration/restful/api/ConfigurationRestfulApi.java b/linkis-public-enhancements/linkis-configuration/src/main/java/org/apache/linkis/configuration/restful/api/ConfigurationRestfulApi.java index 0632ca6aaa..5c235ab269 100644 --- a/linkis-public-enhancements/linkis-configuration/src/main/java/org/apache/linkis/configuration/restful/api/ConfigurationRestfulApi.java +++ b/linkis-public-enhancements/linkis-configuration/src/main/java/org/apache/linkis/configuration/restful/api/ConfigurationRestfulApi.java @@ -180,7 +180,6 @@ public Message getItemList( temp.put("validateRange", configKey.getValidateRange()); temp.put("boundaryType", configKey.getBoundaryType()); temp.put("defaultValue", configKey.getDefaultValue()); - temp.put("description", configKey.getDescription()); // for front-end to judge whether input is required if (StringUtils.isNotEmpty(configKey.getDefaultValue())) { temp.put("require", "true"); diff --git a/linkis-public-enhancements/linkis-configuration/src/main/java/org/apache/linkis/configuration/service/impl/TemplateConfigKeyServiceImpl.java b/linkis-public-enhancements/linkis-configuration/src/main/java/org/apache/linkis/configuration/service/impl/TemplateConfigKeyServiceImpl.java index b990c9ee8e..90c9cc4683 100644 --- a/linkis-public-enhancements/linkis-configuration/src/main/java/org/apache/linkis/configuration/service/impl/TemplateConfigKeyServiceImpl.java +++ b/linkis-public-enhancements/linkis-configuration/src/main/java/org/apache/linkis/configuration/service/impl/TemplateConfigKeyServiceImpl.java @@ -34,7 +34,10 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; +import org.springframework.transaction.PlatformTransactionManager; +import org.springframework.transaction.TransactionStatus; import org.springframework.transaction.annotation.Transactional; +import org.springframework.transaction.support.DefaultTransactionDefinition; import java.text.MessageFormat; import java.util.ArrayList; @@ -63,6 +66,8 @@ public class TemplateConfigKeyServiceImpl implements TemplateConfigKeyService { @Autowired private ConfigKeyLimitForUserMapper configKeyLimitForUserMapper; + @Autowired private PlatformTransactionManager platformTransactionManager; + @Override @Transactional public Boolean updateKeyMapping( @@ -224,6 +229,13 @@ public List queryKeyInfoList(List uuidList) throws Configuration temp.put("validateType", info.getValidateType()); temp.put("validateRange", info.getValidateRange()); temp.put("boundaryType", info.getBoundaryType()); + temp.put("defaultValue", info.getDefaultValue()); + // for front-end to judge whether input is required + if (StringUtils.isNotEmpty(info.getDefaultValue())) { + temp.put("require", "true"); + } else { + temp.put("require", "false"); + } } keys.add(temp); @@ -318,11 +330,21 @@ public Map apply( res.put("msg", "can not get any right key form the db"); errorList.add(res); } else { - configMapper.batchInsertOrUpdateValueList(configValues); - - // batch update user ConfigKeyLimitForUserMapper - configKeyLimitForUserMapper.batchInsertOrUpdateList(configKeyLimitForUsers); + DefaultTransactionDefinition transactionDefinition = new DefaultTransactionDefinition(); + TransactionStatus status = + platformTransactionManager.getTransaction(transactionDefinition); + try { + configMapper.batchInsertOrUpdateValueList(configValues); + // batch update user ConfigKeyLimitForUserMapper + configKeyLimitForUserMapper.batchInsertOrUpdateList(configKeyLimitForUsers); + + platformTransactionManager.commit(status); // commit transaction if everything's fine + } catch (Exception ex) { + platformTransactionManager.rollback( + status); // rollback transaction if any error occurred + throw ex; + } successList.add(res); } diff --git a/linkis-public-enhancements/linkis-jobhistory/src/main/java/org/apache/linkis/jobhistory/cache/impl/DefaultQueryCacheManager.java b/linkis-public-enhancements/linkis-jobhistory/src/main/java/org/apache/linkis/jobhistory/cache/impl/DefaultQueryCacheManager.java index c83d730b8b..7ff5aeb32d 100644 --- a/linkis-public-enhancements/linkis-jobhistory/src/main/java/org/apache/linkis/jobhistory/cache/impl/DefaultQueryCacheManager.java +++ b/linkis-public-enhancements/linkis-jobhistory/src/main/java/org/apache/linkis/jobhistory/cache/impl/DefaultQueryCacheManager.java @@ -195,12 +195,7 @@ public void refreshUndoneTask() { Date sDate = DateUtils.addDays(eDate, -1); queryTasks = jobHistoryMapper.searchWithIdOrderAsc( - undoneTaskMinId, - null, - Arrays.asList("Running", "Inited", "Scheduled"), - sDate, - eDate, - null); + sDate, eDate, undoneTaskMinId, Arrays.asList("Running", "Inited", "Scheduled")); } finally { PageHelper.clearPage(); } diff --git a/linkis-public-enhancements/linkis-jobhistory/src/main/java/org/apache/linkis/jobhistory/dao/JobHistoryMapper.java b/linkis-public-enhancements/linkis-jobhistory/src/main/java/org/apache/linkis/jobhistory/dao/JobHistoryMapper.java index 7bb7656346..c25eee4a2e 100644 --- a/linkis-public-enhancements/linkis-jobhistory/src/main/java/org/apache/linkis/jobhistory/dao/JobHistoryMapper.java +++ b/linkis-public-enhancements/linkis-jobhistory/src/main/java/org/apache/linkis/jobhistory/dao/JobHistoryMapper.java @@ -33,12 +33,10 @@ public interface JobHistoryMapper { void updateJobHistory(JobHistory jobReq); List searchWithIdOrderAsc( - @Param("id") Long id, - @Param("umUser") String username, - @Param("status") List status, @Param("startDate") Date startDate, @Param("endDate") Date endDate, - @Param("engineType") String engineType); + @Param("startId") Long startId, + @Param("status") List status); List search( @Param("id") Long id, diff --git a/linkis-public-enhancements/linkis-jobhistory/src/main/resources/mapper/common/JobHistoryMapper.xml b/linkis-public-enhancements/linkis-jobhistory/src/main/resources/mapper/common/JobHistoryMapper.xml index 824d989c60..9c96c85eb1 100644 --- a/linkis-public-enhancements/linkis-jobhistory/src/main/resources/mapper/common/JobHistoryMapper.xml +++ b/linkis-public-enhancements/linkis-jobhistory/src/main/resources/mapper/common/JobHistoryMapper.xml @@ -98,13 +98,11 @@