From 524117b5caab2c27a27b53b1ec2da67a27383611 Mon Sep 17 00:00:00 2001 From: ChengJie1053 <18033291053@163.com> Date: Wed, 11 Oct 2023 19:49:43 +0800 Subject: [PATCH] Add repl ec to execute Java and scala code --- .../ujes/jdbc/LinkisSQLConnection.scala | 1 + .../manager/am/conf/AMConfiguration.java | 6 +- .../manager/label/conf/LabelCommonConfig.java | 3 + .../label/entity/engine/EngineType.scala | 9 +- .../manager/label/entity/engine/RunType.scala | 1 + .../label/utils/EngineTypeLabelCreator.java | 2 + linkis-engineconn-plugins/pom.xml | 1 + linkis-engineconn-plugins/repl/pom.xml | 127 +++++++++ .../repl/src/main/assembly/distribution.xml | 71 ++++++ .../repl/ReplEngineConnPlugin.java | 72 ++++++ .../ReplProcessEngineConnLaunchBuilder.java | 22 ++ .../repl/conf/ReplConfiguration.java | 38 +++ .../repl/conf/ReplEngineConf.java | 53 ++++ .../engineplugin/repl/conf/ReplType.java | 29 +++ .../repl/errorcode/ReplErrorCodeSummary.java | 52 ++++ .../repl/exception/ReplException.java | 27 ++ .../repl/executor/JavaReplAdapter.java | 30 +++ .../repl/executor/ReplAdapter.java | 23 ++ .../repl/executor/ReplAdapterFactory.java | 44 ++++ .../repl/executor/ReplEngineConnExecutor.java | 240 +++++++++++++++++ .../repl/executor/ScalaReplAdapter.java | 73 ++++++ .../executor/javarepl/JavaReplBuilder.java | 241 ++++++++++++++++++ .../executor/javarepl/JavaReplCompiler.java | 206 +++++++++++++++ .../resources/linkis-engineconn.properties | 23 ++ .../repl/src/main/resources/log4j2.xml | 91 +++++++ .../repl/src/main/resources/repl-ec.md | 117 +++++++++ .../repl/factory/ReplEngineConnFactory.scala | 44 ++++ pom.xml | 4 + 28 files changed, 1644 insertions(+), 6 deletions(-) create mode 100644 linkis-engineconn-plugins/repl/pom.xml create mode 100644 linkis-engineconn-plugins/repl/src/main/assembly/distribution.xml create mode 100644 linkis-engineconn-plugins/repl/src/main/java/org/apache/linkis/engineplugin/repl/ReplEngineConnPlugin.java create mode 100644 linkis-engineconn-plugins/repl/src/main/java/org/apache/linkis/engineplugin/repl/builder/ReplProcessEngineConnLaunchBuilder.java create mode 100644 linkis-engineconn-plugins/repl/src/main/java/org/apache/linkis/engineplugin/repl/conf/ReplConfiguration.java create mode 100644 linkis-engineconn-plugins/repl/src/main/java/org/apache/linkis/engineplugin/repl/conf/ReplEngineConf.java create mode 100644 linkis-engineconn-plugins/repl/src/main/java/org/apache/linkis/engineplugin/repl/conf/ReplType.java create mode 100644 linkis-engineconn-plugins/repl/src/main/java/org/apache/linkis/engineplugin/repl/errorcode/ReplErrorCodeSummary.java create mode 100644 linkis-engineconn-plugins/repl/src/main/java/org/apache/linkis/engineplugin/repl/exception/ReplException.java create mode 100644 linkis-engineconn-plugins/repl/src/main/java/org/apache/linkis/engineplugin/repl/executor/JavaReplAdapter.java create mode 100644 linkis-engineconn-plugins/repl/src/main/java/org/apache/linkis/engineplugin/repl/executor/ReplAdapter.java create mode 100644 linkis-engineconn-plugins/repl/src/main/java/org/apache/linkis/engineplugin/repl/executor/ReplAdapterFactory.java create mode 100644 linkis-engineconn-plugins/repl/src/main/java/org/apache/linkis/engineplugin/repl/executor/ReplEngineConnExecutor.java create mode 100644 linkis-engineconn-plugins/repl/src/main/java/org/apache/linkis/engineplugin/repl/executor/ScalaReplAdapter.java create mode 100644 linkis-engineconn-plugins/repl/src/main/java/org/apache/linkis/engineplugin/repl/executor/javarepl/JavaReplBuilder.java create mode 100644 linkis-engineconn-plugins/repl/src/main/java/org/apache/linkis/engineplugin/repl/executor/javarepl/JavaReplCompiler.java create mode 100644 linkis-engineconn-plugins/repl/src/main/resources/linkis-engineconn.properties create mode 100644 linkis-engineconn-plugins/repl/src/main/resources/log4j2.xml create mode 100644 linkis-engineconn-plugins/repl/src/main/resources/repl-ec.md create mode 100644 linkis-engineconn-plugins/repl/src/main/scala/org/apache/linkis/engineplugin/repl/factory/ReplEngineConnFactory.scala diff --git a/linkis-computation-governance/linkis-jdbc-driver/src/main/scala/org/apache/linkis/ujes/jdbc/LinkisSQLConnection.scala b/linkis-computation-governance/linkis-jdbc-driver/src/main/scala/org/apache/linkis/ujes/jdbc/LinkisSQLConnection.scala index 99bb247738..29c132df93 100644 --- a/linkis-computation-governance/linkis-jdbc-driver/src/main/scala/org/apache/linkis/ujes/jdbc/LinkisSQLConnection.scala +++ b/linkis-computation-governance/linkis-jdbc-driver/src/main/scala/org/apache/linkis/ujes/jdbc/LinkisSQLConnection.scala @@ -442,6 +442,7 @@ class LinkisSQLConnection(private[jdbc] val ujesClient: UJESClient, props: Prope val runType = EngineType.mapStringToEngineType(engine) match { case EngineType.SPARK => RunType.SQL case EngineType.HIVE => RunType.HIVE + case EngineType.REPL => RunType.REPL case EngineType.TRINO => RunType.TRINO_SQL case EngineType.PRESTO => RunType.PRESTO_SQL case EngineType.NEBULA => RunType.NEBULA_SQL diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/conf/AMConfiguration.java b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/conf/AMConfiguration.java index 0f018ca9de..0c95cef777 100644 --- a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/conf/AMConfiguration.java +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/conf/AMConfiguration.java @@ -69,7 +69,7 @@ public class AMConfiguration { public static final CommonVars MULTI_USER_ENGINE_TYPES = CommonVars.apply( "wds.linkis.multi.user.engine.types", - "jdbc,es,presto,io_file,appconn,openlookeng,trino,nebula,hbase"); + "jdbc,es,presto,io_file,appconn,openlookeng,trino,nebula,hbase,repl"); public static final CommonVars ALLOW_BATCH_KILL_ENGINE_TYPES = CommonVars.apply("wds.linkis.allow.batch.kill.engine.types", "spark,hive,python"); @@ -105,8 +105,8 @@ public class AMConfiguration { public static String getDefaultMultiEngineUser() { String jvmUser = Utils.getJvmUser(); return String.format( - "{jdbc:\"%s\", es: \"%s\", presto:\"%s\", appconn:\"%s\", openlookeng:\"%s\", trino:\"%s\", nebula:\"%s\", hbase:\"%s\",io_file:\"root\"}", - jvmUser, jvmUser, jvmUser, jvmUser, jvmUser, jvmUser, jvmUser, jvmUser); + "{jdbc:\"%s\", es: \"%s\", presto:\"%s\", appconn:\"%s\", openlookeng:\"%s\", trino:\"%s\", nebula:\"%s\",repl:\"%s\", hbase:\"%s\",io_file:\"root\"}", + jvmUser, jvmUser, jvmUser, jvmUser, jvmUser, jvmUser, jvmUser, jvmUser, jvmUser); } public static boolean isMultiUserEngine(String engineType) { diff --git a/linkis-computation-governance/linkis-manager/linkis-label-common/src/main/java/org/apache/linkis/manager/label/conf/LabelCommonConfig.java b/linkis-computation-governance/linkis-manager/linkis-label-common/src/main/java/org/apache/linkis/manager/label/conf/LabelCommonConfig.java index f0f96ba1f1..fbb4e5d38c 100644 --- a/linkis-computation-governance/linkis-manager/linkis-label-common/src/main/java/org/apache/linkis/manager/label/conf/LabelCommonConfig.java +++ b/linkis-computation-governance/linkis-manager/linkis-label-common/src/main/java/org/apache/linkis/manager/label/conf/LabelCommonConfig.java @@ -42,6 +42,9 @@ public class LabelCommonConfig { public static final CommonVars PYTHON_ENGINE_VERSION = CommonVars.apply("wds.linkis.python.engine.version", "python2"); + public static final CommonVars REPL_ENGINE_VERSION = + CommonVars.apply("wds.linkis.repl.engine.version", "1"); + public static final CommonVars FILE_ENGINE_VERSION = CommonVars.apply("wds.linkis.file.engine.version", "1.0"); diff --git a/linkis-computation-governance/linkis-manager/linkis-label-common/src/main/java/org/apache/linkis/manager/label/entity/engine/EngineType.scala b/linkis-computation-governance/linkis-manager/linkis-label-common/src/main/java/org/apache/linkis/manager/label/entity/engine/EngineType.scala index 27540fac68..c5c44017e6 100644 --- a/linkis-computation-governance/linkis-manager/linkis-label-common/src/main/java/org/apache/linkis/manager/label/entity/engine/EngineType.scala +++ b/linkis-computation-governance/linkis-manager/linkis-label-common/src/main/java/org/apache/linkis/manager/label/entity/engine/EngineType.scala @@ -5,16 +5,16 @@ * The ASF licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at - * + * * http://www.apache.org/licenses/LICENSE-2.0 - * + * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ - + package org.apache.linkis.manager.label.entity.engine import org.apache.linkis.common.utils.Logging @@ -33,6 +33,8 @@ object EngineType extends Enumeration with Logging { val PYTHON = Value("python") + val REPL = Value("repl") + val SHELL = Value("shell") val JDBC = Value("jdbc") @@ -95,6 +97,7 @@ object EngineType extends Enumeration with Logging { case _ if PIPELINE.toString.equalsIgnoreCase(str) => PIPELINE case _ if PRESTO.toString.equalsIgnoreCase(str) => PRESTO case _ if NEBULA.toString.equalsIgnoreCase(str) => NEBULA + case _ if REPL.toString.equalsIgnoreCase(str) => REPL case _ if FLINK.toString.equalsIgnoreCase(str) => FLINK case _ if APPCONN.toString.equals(str) => APPCONN case _ if SQOOP.toString.equalsIgnoreCase(str) => SQOOP diff --git a/linkis-computation-governance/linkis-manager/linkis-label-common/src/main/java/org/apache/linkis/manager/label/entity/engine/RunType.scala b/linkis-computation-governance/linkis-manager/linkis-label-common/src/main/java/org/apache/linkis/manager/label/entity/engine/RunType.scala index abb3e010f8..645c15dbdb 100644 --- a/linkis-computation-governance/linkis-manager/linkis-label-common/src/main/java/org/apache/linkis/manager/label/entity/engine/RunType.scala +++ b/linkis-computation-governance/linkis-manager/linkis-label-common/src/main/java/org/apache/linkis/manager/label/entity/engine/RunType.scala @@ -24,6 +24,7 @@ object RunType extends Enumeration { val HIVE = Value("hql") val SCALA = Value("scala") val PYTHON = Value("python") + val REPL = Value("repl") val JAVA = Value("java") val PYSPARK = Value("py") val R = Value("r") diff --git a/linkis-computation-governance/linkis-manager/linkis-label-common/src/main/java/org/apache/linkis/manager/label/utils/EngineTypeLabelCreator.java b/linkis-computation-governance/linkis-manager/linkis-label-common/src/main/java/org/apache/linkis/manager/label/utils/EngineTypeLabelCreator.java index 40fd7c78b1..d7911c030e 100644 --- a/linkis-computation-governance/linkis-manager/linkis-label-common/src/main/java/org/apache/linkis/manager/label/utils/EngineTypeLabelCreator.java +++ b/linkis-computation-governance/linkis-manager/linkis-label-common/src/main/java/org/apache/linkis/manager/label/utils/EngineTypeLabelCreator.java @@ -50,6 +50,8 @@ private static void init() { EngineType.HIVE().toString(), LabelCommonConfig.HIVE_ENGINE_VERSION.getValue()); defaultVersion.put( EngineType.PYTHON().toString(), LabelCommonConfig.PYTHON_ENGINE_VERSION.getValue()); + defaultVersion.put( + EngineType.REPL().toString(), LabelCommonConfig.REPL_ENGINE_VERSION.getValue()); defaultVersion.put( EngineType.IO_ENGINE_FILE().toString(), LabelCommonConfig.FILE_ENGINE_VERSION.getValue()); diff --git a/linkis-engineconn-plugins/pom.xml b/linkis-engineconn-plugins/pom.xml index 30ee76662a..5df6526748 100644 --- a/linkis-engineconn-plugins/pom.xml +++ b/linkis-engineconn-plugins/pom.xml @@ -42,6 +42,7 @@ elasticsearch seatunnel hbase + repl diff --git a/linkis-engineconn-plugins/repl/pom.xml b/linkis-engineconn-plugins/repl/pom.xml new file mode 100644 index 0000000000..573cb85576 --- /dev/null +++ b/linkis-engineconn-plugins/repl/pom.xml @@ -0,0 +1,127 @@ + + + + 4.0.0 + + org.apache.linkis + linkis + ${revision} + ../../pom.xml + + + linkis-engineplugin-repl + + + + org.apache.linkis + linkis-engineconn-plugin-core + ${project.version} + + + + org.apache.linkis + linkis-computation-engineconn + ${project.version} + + + + org.apache.linkis + linkis-storage + ${project.version} + provided + + + + org.apache.linkis + linkis-rpc + ${project.version} + provided + + + + org.apache.linkis + linkis-common + ${project.version} + provided + + + + + org.javassist + javassist + ${javassist.version} + + + + + org.scala-lang + scala-library + provided + + + org.scala-lang + scala-compiler + provided + + + org.scala-lang + scala-reflect + provided + + + + + + + + net.alchim31.maven + scala-maven-plugin + + + + org.apache.maven.plugins + maven-assembly-plugin + false + + false + out + false + false + + src/main/assembly/distribution.xml + + + + + make-assembly + + single + + package + + + src/main/assembly/distribution.xml + + + + + + + + + diff --git a/linkis-engineconn-plugins/repl/src/main/assembly/distribution.xml b/linkis-engineconn-plugins/repl/src/main/assembly/distribution.xml new file mode 100644 index 0000000000..3216ffa68b --- /dev/null +++ b/linkis-engineconn-plugins/repl/src/main/assembly/distribution.xml @@ -0,0 +1,71 @@ + + + + + linkis-engineplugin-repl + + dir + zip + + true + repl + + + + + + /dist/${repl.version}/lib + true + true + false + false + true + + + + + + + + ${basedir}/src/main/resources + + linkis-engineconn.properties + log4j2.xml + + 0777 + dist/${repl.version}/conf + unix + + + + ${basedir}/target + + *.jar + + + *doc.jar + + 0777 + plugin/${repl.version} + + + + + + diff --git a/linkis-engineconn-plugins/repl/src/main/java/org/apache/linkis/engineplugin/repl/ReplEngineConnPlugin.java b/linkis-engineconn-plugins/repl/src/main/java/org/apache/linkis/engineplugin/repl/ReplEngineConnPlugin.java new file mode 100644 index 0000000000..26d21bdca6 --- /dev/null +++ b/linkis-engineconn-plugins/repl/src/main/java/org/apache/linkis/engineplugin/repl/ReplEngineConnPlugin.java @@ -0,0 +1,72 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.engineplugin.repl; + +import org.apache.linkis.engineplugin.repl.builder.ReplProcessEngineConnLaunchBuilder; +import org.apache.linkis.engineplugin.repl.factory.ReplEngineConnFactory; +import org.apache.linkis.manager.engineplugin.common.EngineConnPlugin; +import org.apache.linkis.manager.engineplugin.common.creation.EngineConnFactory; +import org.apache.linkis.manager.engineplugin.common.launch.EngineConnLaunchBuilder; +import org.apache.linkis.manager.engineplugin.common.resource.EngineResourceFactory; +import org.apache.linkis.manager.engineplugin.common.resource.GenericEngineResourceFactory; +import org.apache.linkis.manager.label.entity.Label; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +public class ReplEngineConnPlugin implements EngineConnPlugin { + private Object resourceLocker = new Object(); + private Object engineFactoryLocker = new Object(); + private volatile EngineResourceFactory engineResourceFactory; + private volatile EngineConnFactory engineFactory; + private List> defaultLabels = new ArrayList<>(); + + @Override + public void init(Map params) {} + + @Override + public EngineResourceFactory getEngineResourceFactory() { + if (null == engineResourceFactory) { + synchronized (resourceLocker) { + engineResourceFactory = new GenericEngineResourceFactory(); + } + } + return engineResourceFactory; + } + + @Override + public EngineConnLaunchBuilder getEngineConnLaunchBuilder() { + return new ReplProcessEngineConnLaunchBuilder(); + } + + @Override + public EngineConnFactory getEngineConnFactory() { + if (null == engineFactory) { + synchronized (engineFactoryLocker) { + engineFactory = new ReplEngineConnFactory(); + } + } + return engineFactory; + } + + @Override + public List> getDefaultLabels() { + return defaultLabels; + } +} diff --git a/linkis-engineconn-plugins/repl/src/main/java/org/apache/linkis/engineplugin/repl/builder/ReplProcessEngineConnLaunchBuilder.java b/linkis-engineconn-plugins/repl/src/main/java/org/apache/linkis/engineplugin/repl/builder/ReplProcessEngineConnLaunchBuilder.java new file mode 100644 index 0000000000..d528393f81 --- /dev/null +++ b/linkis-engineconn-plugins/repl/src/main/java/org/apache/linkis/engineplugin/repl/builder/ReplProcessEngineConnLaunchBuilder.java @@ -0,0 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.engineplugin.repl.builder; + +import org.apache.linkis.manager.engineplugin.common.launch.process.JavaProcessEngineConnLaunchBuilder; + +public class ReplProcessEngineConnLaunchBuilder extends JavaProcessEngineConnLaunchBuilder {} diff --git a/linkis-engineconn-plugins/repl/src/main/java/org/apache/linkis/engineplugin/repl/conf/ReplConfiguration.java b/linkis-engineconn-plugins/repl/src/main/java/org/apache/linkis/engineplugin/repl/conf/ReplConfiguration.java new file mode 100644 index 0000000000..d4f8cd2816 --- /dev/null +++ b/linkis-engineconn-plugins/repl/src/main/java/org/apache/linkis/engineplugin/repl/conf/ReplConfiguration.java @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.engineplugin.repl.conf; + +import org.apache.linkis.common.conf.CommonVars; + +public class ReplConfiguration { + + public static final CommonVars ENGINE_CONCURRENT_LIMIT = + CommonVars.apply("linkis.engineconn.concurrent.limit", 100); + + public static final CommonVars REPL_TYPE = + CommonVars.apply("linkis.repl.type", ReplType.JAVA); + + public static final CommonVars CLASSPATH_DIR = + CommonVars.apply("linkis.repl.classpath.dir", ""); + + public static final CommonVars METHOD_NAME = + CommonVars.apply("linkis.repl.method.name", ""); + + public static final CommonVars ENGINE_DEFAULT_LIMIT = + CommonVars.apply("linkis.repl.default.limit", 5000); +} diff --git a/linkis-engineconn-plugins/repl/src/main/java/org/apache/linkis/engineplugin/repl/conf/ReplEngineConf.java b/linkis-engineconn-plugins/repl/src/main/java/org/apache/linkis/engineplugin/repl/conf/ReplEngineConf.java new file mode 100644 index 0000000000..4077db48cc --- /dev/null +++ b/linkis-engineconn-plugins/repl/src/main/java/org/apache/linkis/engineplugin/repl/conf/ReplEngineConf.java @@ -0,0 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.engineplugin.repl.conf; + +import org.apache.linkis.common.conf.Configuration; +import org.apache.linkis.governance.common.protocol.conf.RequestQueryEngineConfigWithGlobalConfig; +import org.apache.linkis.governance.common.protocol.conf.ResponseQueryConfig; +import org.apache.linkis.manager.label.entity.engine.EngineTypeLabel; +import org.apache.linkis.manager.label.entity.engine.UserCreatorLabel; +import org.apache.linkis.protocol.CacheableProtocol; +import org.apache.linkis.rpc.RPCMapCache; + +import java.util.Map; + +import scala.Tuple2; + +public class ReplEngineConf + extends RPCMapCache, String, String> { + + public ReplEngineConf() { + super(Configuration.CLOUD_CONSOLE_CONFIGURATION_SPRING_APPLICATION_NAME().getValue()); + } + + @Override + public CacheableProtocol createRequest(Tuple2 labelTuple) { + return new RequestQueryEngineConfigWithGlobalConfig(labelTuple._1(), labelTuple._2(), null); + } + + @Override + public Map createMap(Object obj) { + if (obj instanceof ResponseQueryConfig) { + ResponseQueryConfig response = (ResponseQueryConfig) obj; + return response.getKeyAndValue(); + } else { + return null; + } + } +} diff --git a/linkis-engineconn-plugins/repl/src/main/java/org/apache/linkis/engineplugin/repl/conf/ReplType.java b/linkis-engineconn-plugins/repl/src/main/java/org/apache/linkis/engineplugin/repl/conf/ReplType.java new file mode 100644 index 0000000000..b1061ace25 --- /dev/null +++ b/linkis-engineconn-plugins/repl/src/main/java/org/apache/linkis/engineplugin/repl/conf/ReplType.java @@ -0,0 +1,29 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.engineplugin.repl.conf; + +public class ReplType { + + public static final String JAVA = "java"; + + public static final String SCALA = "scala"; + + public static boolean isSupportReplType(String type) { + return JAVA.equalsIgnoreCase(type) || SCALA.equalsIgnoreCase(type); + } +} diff --git a/linkis-engineconn-plugins/repl/src/main/java/org/apache/linkis/engineplugin/repl/errorcode/ReplErrorCodeSummary.java b/linkis-engineconn-plugins/repl/src/main/java/org/apache/linkis/engineplugin/repl/errorcode/ReplErrorCodeSummary.java new file mode 100644 index 0000000000..7daf0c8887 --- /dev/null +++ b/linkis-engineconn-plugins/repl/src/main/java/org/apache/linkis/engineplugin/repl/errorcode/ReplErrorCodeSummary.java @@ -0,0 +1,52 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.engineplugin.repl.errorcode; + +import org.apache.linkis.common.errorcode.ErrorCodeUtils; +import org.apache.linkis.common.errorcode.LinkisErrorCode; + +public enum ReplErrorCodeSummary implements LinkisErrorCode { + NOT_SUPPORT_REPL_TYPE(29001, "Repl engineplugin does not support this type(Repl引擎不支持这个类型)"), + REPL_CODE_IS_NOT_BLANK(29002, "Repl engine code cannot be empty(Repl引擎代码不能为空)"), + + UNABLE_RESOLVE_JAVA_METHOD_NAME( + 29003, "Repl engine unable to resolve java method name(Repl引擎无法解析java方法名称)"), + + REPL_SCALA_TASK_EXECUTOR_FAILED( + 29004, "Repl engine scala task executor failed(Repl引擎scala任务执行失败)"); + + private final int errorCode; + + private final String errorDesc; + + ReplErrorCodeSummary(int errorCode, String errorDesc) { + ErrorCodeUtils.validateErrorCode(errorCode, 26000, 29999); + this.errorCode = errorCode; + this.errorDesc = errorDesc; + } + + @Override + public int getErrorCode() { + return errorCode; + } + + @Override + public String getErrorDesc() { + return errorDesc; + } +} diff --git a/linkis-engineconn-plugins/repl/src/main/java/org/apache/linkis/engineplugin/repl/exception/ReplException.java b/linkis-engineconn-plugins/repl/src/main/java/org/apache/linkis/engineplugin/repl/exception/ReplException.java new file mode 100644 index 0000000000..ab2fa13ab3 --- /dev/null +++ b/linkis-engineconn-plugins/repl/src/main/java/org/apache/linkis/engineplugin/repl/exception/ReplException.java @@ -0,0 +1,27 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.engineplugin.repl.exception; + +import org.apache.linkis.common.exception.ErrorException; + +public class ReplException extends ErrorException { + + public ReplException(int errorCode, String message) { + super(errorCode, message); + } +} diff --git a/linkis-engineconn-plugins/repl/src/main/java/org/apache/linkis/engineplugin/repl/executor/JavaReplAdapter.java b/linkis-engineconn-plugins/repl/src/main/java/org/apache/linkis/engineplugin/repl/executor/JavaReplAdapter.java new file mode 100644 index 0000000000..b42219316f --- /dev/null +++ b/linkis-engineconn-plugins/repl/src/main/java/org/apache/linkis/engineplugin/repl/executor/JavaReplAdapter.java @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.engineplugin.repl.executor; + +import org.apache.linkis.engineplugin.repl.executor.javarepl.JavaReplCompiler; + +public class JavaReplAdapter extends ReplAdapter { + @Override + public void executorCode(String code, String classpathDir, String methodName) throws Exception { + JavaReplCompiler compiler = new JavaReplCompiler(); + + compiler.compileAndExecutor( + code, Thread.currentThread().getContextClassLoader(), classpathDir, methodName); + } +} diff --git a/linkis-engineconn-plugins/repl/src/main/java/org/apache/linkis/engineplugin/repl/executor/ReplAdapter.java b/linkis-engineconn-plugins/repl/src/main/java/org/apache/linkis/engineplugin/repl/executor/ReplAdapter.java new file mode 100644 index 0000000000..d5607b06c9 --- /dev/null +++ b/linkis-engineconn-plugins/repl/src/main/java/org/apache/linkis/engineplugin/repl/executor/ReplAdapter.java @@ -0,0 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.engineplugin.repl.executor; + +public abstract class ReplAdapter { + public abstract void executorCode(String code, String classpathDir, String methodName) + throws Exception; +} diff --git a/linkis-engineconn-plugins/repl/src/main/java/org/apache/linkis/engineplugin/repl/executor/ReplAdapterFactory.java b/linkis-engineconn-plugins/repl/src/main/java/org/apache/linkis/engineplugin/repl/executor/ReplAdapterFactory.java new file mode 100644 index 0000000000..4dddffbf7f --- /dev/null +++ b/linkis-engineconn-plugins/repl/src/main/java/org/apache/linkis/engineplugin/repl/executor/ReplAdapterFactory.java @@ -0,0 +1,44 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.engineplugin.repl.executor; + +import org.apache.linkis.engineplugin.repl.conf.ReplType; +import org.apache.linkis.engineplugin.repl.errorcode.ReplErrorCodeSummary; +import org.apache.linkis.engineplugin.repl.exception.ReplException; + +import org.apache.commons.lang3.StringUtils; + +public class ReplAdapterFactory { + + public static ReplAdapter create(String replType) { + if (StringUtils.isBlank(replType) || !ReplType.isSupportReplType(replType)) { + throw new ReplException( + ReplErrorCodeSummary.NOT_SUPPORT_REPL_TYPE.getErrorCode(), + ReplErrorCodeSummary.NOT_SUPPORT_REPL_TYPE.getErrorDesc()); + } + + ReplAdapter replAdapter = null; + + if (ReplType.JAVA.equalsIgnoreCase(replType)) { + replAdapter = new JavaReplAdapter(); + } else if (ReplType.SCALA.equalsIgnoreCase(replType)) { + replAdapter = new ScalaReplAdapter(); + } + return replAdapter; + } +} diff --git a/linkis-engineconn-plugins/repl/src/main/java/org/apache/linkis/engineplugin/repl/executor/ReplEngineConnExecutor.java b/linkis-engineconn-plugins/repl/src/main/java/org/apache/linkis/engineplugin/repl/executor/ReplEngineConnExecutor.java new file mode 100644 index 0000000000..f18714132c --- /dev/null +++ b/linkis-engineconn-plugins/repl/src/main/java/org/apache/linkis/engineplugin/repl/executor/ReplEngineConnExecutor.java @@ -0,0 +1,240 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.engineplugin.repl.executor; + +import org.apache.linkis.common.log.LogUtils; +import org.apache.linkis.common.utils.OverloadUtils; +import org.apache.linkis.engineconn.computation.executor.entity.EngineConnTask; +import org.apache.linkis.engineconn.computation.executor.execute.ConcurrentComputationExecutor; +import org.apache.linkis.engineconn.computation.executor.execute.EngineExecutionContext; +import org.apache.linkis.engineconn.core.EngineConnObject; +import org.apache.linkis.engineplugin.repl.conf.ReplConfiguration; +import org.apache.linkis.engineplugin.repl.conf.ReplEngineConf; +import org.apache.linkis.engineplugin.repl.errorcode.ReplErrorCodeSummary; +import org.apache.linkis.engineplugin.repl.exception.ReplException; +import org.apache.linkis.manager.common.entity.resource.CommonNodeResource; +import org.apache.linkis.manager.common.entity.resource.LoadResource; +import org.apache.linkis.manager.common.entity.resource.NodeResource; +import org.apache.linkis.manager.engineplugin.common.util.NodeResourceUtils; +import org.apache.linkis.manager.label.entity.Label; +import org.apache.linkis.manager.label.entity.engine.EngineTypeLabel; +import org.apache.linkis.manager.label.entity.engine.UserCreatorLabel; +import org.apache.linkis.protocol.engine.JobProgressInfo; +import org.apache.linkis.rpc.Sender; +import org.apache.linkis.scheduler.executer.ExecuteResponse; +import org.apache.linkis.scheduler.executer.SuccessExecuteResponse; + +import org.apache.commons.collections4.MapUtils; +import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.exception.ExceptionUtils; + +import org.springframework.util.CollectionUtils; + +import java.util.*; +import java.util.concurrent.ConcurrentHashMap; + +import scala.Tuple2; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class ReplEngineConnExecutor extends ConcurrentComputationExecutor { + + private static final Logger logger = LoggerFactory.getLogger(ReplEngineConnExecutor.class); + private int id; + + private ReplAdapter replAdapter; + private List> executorLabels = new ArrayList<>(2); + private Map threadCache = new ConcurrentHashMap<>(); + + private Map configMap = new HashMap<>(); + + public ReplEngineConnExecutor(int outputPrintLimit, int id) { + super(outputPrintLimit); + this.id = id; + } + + @Override + public void init() { + super.init(); + } + + @Override + public ExecuteResponse execute(EngineConnTask engineConnTask) { + Optional> userCreatorLabelOp = + Arrays.stream(engineConnTask.getLables()) + .filter(label -> label instanceof UserCreatorLabel) + .findFirst(); + Optional> engineTypeLabelOp = + Arrays.stream(engineConnTask.getLables()) + .filter(label -> label instanceof EngineTypeLabel) + .findFirst(); + + if (userCreatorLabelOp.isPresent() && engineTypeLabelOp.isPresent()) { + UserCreatorLabel userCreatorLabel = (UserCreatorLabel) userCreatorLabelOp.get(); + EngineTypeLabel engineTypeLabel = (EngineTypeLabel) engineTypeLabelOp.get(); + + Map cacheMap = + new ReplEngineConf().getCacheMap(new Tuple2<>(userCreatorLabel, engineTypeLabel)); + if (MapUtils.isNotEmpty(cacheMap)) { + configMap.putAll(cacheMap); + } + } + + Map taskParams = engineConnTask.getProperties(); + + if (MapUtils.isNotEmpty(taskParams)) { + taskParams.entrySet().stream() + .filter(entry -> entry.getValue() != null) + .forEach(entry -> configMap.put(entry.getKey(), String.valueOf(entry.getValue()))); + } + + String replType = ReplConfiguration.REPL_TYPE.getValue(configMap); + + replAdapter = ReplAdapterFactory.create(replType); + return super.execute(engineConnTask); + } + + @Override + public ExecuteResponse executeLine(EngineExecutionContext engineExecutorContext, String code) { + String realCode; + if (StringUtils.isBlank(code)) { + throw new ReplException( + ReplErrorCodeSummary.REPL_CODE_IS_NOT_BLANK.getErrorCode(), + ReplErrorCodeSummary.REPL_CODE_IS_NOT_BLANK.getErrorDesc()); + } else { + realCode = code.trim(); + } + logger.info("Repl engine begins to run code:\n {}", realCode); + + String taskId = engineExecutorContext.getJobId().get(); + + initialStatusUpdates(taskId, engineExecutorContext); + + String classpathDir = ReplConfiguration.CLASSPATH_DIR.getValue(configMap); + + String methodName = ReplConfiguration.METHOD_NAME.getValue(configMap); + + Thread thread = + new Thread( + () -> { + try { + replAdapter.executorCode(realCode, classpathDir, methodName); + } catch (Exception e) { + String errorMessage = ExceptionUtils.getStackTrace(e); + logger.error("Repl engine execute failed : {}", errorMessage); + engineExecutorContext.appendStdout(LogUtils.generateERROR(errorMessage)); + } + }); + + thread.start(); + threadCache.put(taskId, thread); + return new SuccessExecuteResponse(); + } + + @Override + public ExecuteResponse executeCompletely( + EngineExecutionContext engineExecutorContext, String code, String completedLine) { + return null; + } + + @Override + public float progress(String taskID) { + return 0.0f; + } + + @Override + public JobProgressInfo[] getProgressInfo(String taskID) { + return new JobProgressInfo[0]; + } + + @Override + public void killTask(String taskId) { + Thread thread = threadCache.remove(taskId); + if (null != thread) { + thread.interrupt(); + } + super.killTask(taskId); + } + + @Override + public List> getExecutorLabels() { + return executorLabels; + } + + @Override + public void setExecutorLabels(List> labels) { + if (!CollectionUtils.isEmpty(labels)) { + executorLabels.clear(); + executorLabels.addAll(labels); + } + } + + @Override + public boolean supportCallBackLogs() { + return false; + } + + @Override + public NodeResource requestExpectedResource(NodeResource expectedResource) { + return null; + } + + @Override + public NodeResource getCurrentNodeResource() { + NodeResourceUtils.appendMemoryUnitIfMissing( + EngineConnObject.getEngineCreationContext().getOptions()); + + CommonNodeResource resource = new CommonNodeResource(); + LoadResource usedResource = new LoadResource(OverloadUtils.getProcessMaxMemory(), 1); + resource.setUsedResource(usedResource); + return resource; + } + + @Override + public String getId() { + return Sender.getThisServiceInstance().getInstance() + "_" + id; + } + + @Override + public int getConcurrentLimit() { + return ReplConfiguration.ENGINE_CONCURRENT_LIMIT.getValue(); + } + + private void initialStatusUpdates(String taskId, EngineExecutionContext engineExecutorContext) { + engineExecutorContext.pushProgress(progress(taskId), getProgressInfo(taskId)); + } + + @Override + public void killAll() { + Iterator iterator = threadCache.values().iterator(); + while (iterator.hasNext()) { + Thread thread = iterator.next(); + if (thread != null) { + thread.interrupt(); + } + } + threadCache.clear(); + } + + @Override + public void close() { + killAll(); + super.close(); + } +} diff --git a/linkis-engineconn-plugins/repl/src/main/java/org/apache/linkis/engineplugin/repl/executor/ScalaReplAdapter.java b/linkis-engineconn-plugins/repl/src/main/java/org/apache/linkis/engineplugin/repl/executor/ScalaReplAdapter.java new file mode 100644 index 0000000000..7c5349c680 --- /dev/null +++ b/linkis-engineconn-plugins/repl/src/main/java/org/apache/linkis/engineplugin/repl/executor/ScalaReplAdapter.java @@ -0,0 +1,73 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.engineplugin.repl.executor; + +import org.apache.linkis.engineplugin.repl.errorcode.ReplErrorCodeSummary; +import org.apache.linkis.engineplugin.repl.exception.ReplException; + +import org.apache.commons.io.FileUtils; +import org.apache.commons.lang3.StringUtils; + +import java.io.*; + +import scala.tools.nsc.Settings; +import scala.tools.nsc.interpreter.ILoop; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class ScalaReplAdapter extends ReplAdapter { + + private static final Logger logger = LoggerFactory.getLogger(ScalaReplAdapter.class); + + @Override + public void executorCode(String code, String classpathDir, String methodName) { + StringReader stringReader = new StringReader(code); + StringWriter stringWriter = new StringWriter(); + + ILoop repl = new ILoop(new BufferedReader(stringReader), new PrintWriter(stringWriter)); + + try { + logger.info("Scala repl start executor"); + + Settings settings = new Settings(); + settings.usejavacp().tryToSetFromPropertyValue("true"); + + if (StringUtils.isNotBlank(classpathDir) && FileUtils.isDirectory(new File(classpathDir))) { + settings.classpath().value_$eq(classpathDir); + } + + boolean process = repl.process(settings); + + String scalaReplLog = stringWriter.toString(); + logger.info("Scala repl log: {}", scalaReplLog); + + if (process) { + logger.info("Scala repl executor success"); + } else { + logger.error("Scala repl executor failed"); + throw new ReplException( + ReplErrorCodeSummary.REPL_SCALA_TASK_EXECUTOR_FAILED.getErrorCode(), + ReplErrorCodeSummary.REPL_SCALA_TASK_EXECUTOR_FAILED.getErrorDesc()); + } + + } finally { + repl.closeInterpreter(); + } + } +} diff --git a/linkis-engineconn-plugins/repl/src/main/java/org/apache/linkis/engineplugin/repl/executor/javarepl/JavaReplBuilder.java b/linkis-engineconn-plugins/repl/src/main/java/org/apache/linkis/engineplugin/repl/executor/javarepl/JavaReplBuilder.java new file mode 100644 index 0000000000..f70821c4b3 --- /dev/null +++ b/linkis-engineconn-plugins/repl/src/main/java/org/apache/linkis/engineplugin/repl/executor/javarepl/JavaReplBuilder.java @@ -0,0 +1,241 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.engineplugin.repl.executor.javarepl; + +import org.apache.commons.lang3.StringUtils; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import javassist.*; + +public class JavaReplBuilder { + + private String className; + + private String superClassName = "java.lang.Object"; + + private final List imports = new ArrayList<>(); + + private final Map fullNames = new HashMap<>(); + + private final List ifaces = new ArrayList<>(); + + private final List constructors = new ArrayList<>(); + + private final List fields = new ArrayList<>(); + + private final List methods = new ArrayList<>(); + + public String getClassName() { + return className; + } + + public void setClassName(String className) { + this.className = className; + } + + public String getSuperClassName() { + return superClassName; + } + + public void setSuperClassName(String superClassName) { + this.superClassName = getQualifiedClassName(superClassName); + } + + public List getImports() { + return imports; + } + + public void addImports(String pkg) { + int pi = pkg.lastIndexOf('.'); + if (pi > 0) { + String pkgName = pkg.substring(0, pi); + this.imports.add(pkgName); + if (!pkg.endsWith(".*")) { + fullNames.put(pkg.substring(pi + 1), pkg); + } + } + } + + public List getInterfaces() { + return ifaces; + } + + public void addInterface(String iface) { + this.ifaces.add(getQualifiedClassName(iface)); + } + + public List getConstructors() { + return constructors; + } + + public void addConstructor(String constructor) { + this.constructors.add(constructor); + } + + public List getFields() { + return fields; + } + + public void addField(String field) { + this.fields.add(field); + } + + public List getMethods() { + return methods; + } + + public void addMethod(String method) { + this.methods.add(method); + } + + /** + * get full qualified class name + * + * @param className super class name, maybe qualified or not + */ + protected String getQualifiedClassName(String className) { + if (className.contains(".")) { + return className; + } + + if (fullNames.containsKey(className)) { + return fullNames.get(className); + } + + return forName(imports.toArray(new String[0]), className).getName(); + } + + /** build CtClass object */ + public CtClass build(ClassLoader classLoader) throws NotFoundException, CannotCompileException { + ClassPool pool = new ClassPool(true); + pool.insertClassPath(new LoaderClassPath(classLoader)); + + // create class + CtClass ctClass = null; + + if (StringUtils.isNotBlank(superClassName)) { + ctClass = pool.makeClass(className, pool.get(superClassName)); + } else { + ctClass = pool.makeClass(className); + } + + // add imported packages + imports.forEach(pool::importPackage); + + // add implemented interfaces + for (String iface : ifaces) { + ctClass.addInterface(pool.get(iface)); + } + + // add constructors + for (String constructor : constructors) { + ctClass.addConstructor(CtNewConstructor.make(constructor, ctClass)); + } + + // add fields + for (String field : fields) { + ctClass.addField(CtField.make(field, ctClass)); + } + + // add methods + for (String method : methods) { + ctClass.addMethod(CtNewMethod.make(method, ctClass)); + } + + return ctClass; + } + + public static Class forName(String[] packages, String className) { + try { + return classForName(className); + } catch (ClassNotFoundException e) { + if (packages != null && packages.length > 0) { + for (String pkg : packages) { + try { + return classForName(pkg + "." + className); + } catch (ClassNotFoundException ignore) { + } + } + } + throw new IllegalStateException(e.getMessage(), e); + } + } + + public static Class classForName(String className) throws ClassNotFoundException { + switch (className) { + case "boolean": + return boolean.class; + case "byte": + return byte.class; + case "char": + return char.class; + case "short": + return short.class; + case "int": + return int.class; + case "long": + return long.class; + case "float": + return float.class; + case "double": + return double.class; + case "boolean[]": + return boolean[].class; + case "byte[]": + return byte[].class; + case "char[]": + return char[].class; + case "short[]": + return short[].class; + case "int[]": + return int[].class; + case "long[]": + return long[].class; + case "float[]": + return float[].class; + case "double[]": + return double[].class; + default: + } + try { + return arrayForName(className); + } catch (ClassNotFoundException e) { + // try to load from java.lang package + if (className.indexOf('.') == -1) { + try { + return arrayForName("java.lang." + className); + } catch (ClassNotFoundException ignore) { + // ignore, let the original exception be thrown + } + } + throw e; + } + } + + private static Class arrayForName(String className) throws ClassNotFoundException { + return Class.forName( + className.endsWith("[]") + ? "[L" + className.substring(0, className.length() - 2) + ";" + : className, + true, + Thread.currentThread().getContextClassLoader()); + } +} diff --git a/linkis-engineconn-plugins/repl/src/main/java/org/apache/linkis/engineplugin/repl/executor/javarepl/JavaReplCompiler.java b/linkis-engineconn-plugins/repl/src/main/java/org/apache/linkis/engineplugin/repl/executor/javarepl/JavaReplCompiler.java new file mode 100644 index 0000000000..7be4369735 --- /dev/null +++ b/linkis-engineconn-plugins/repl/src/main/java/org/apache/linkis/engineplugin/repl/executor/javarepl/JavaReplCompiler.java @@ -0,0 +1,206 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.engineplugin.repl.executor.javarepl; + +import org.apache.linkis.engineplugin.repl.errorcode.ReplErrorCodeSummary; +import org.apache.linkis.engineplugin.repl.exception.ReplException; + +import org.apache.commons.io.FileUtils; +import org.apache.commons.lang3.ArrayUtils; +import org.apache.commons.lang3.RandomStringUtils; +import org.apache.commons.lang3.StringUtils; + +import java.io.File; +import java.util.Arrays; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import javassist.ClassPool; +import javassist.CtClass; +import javassist.CtMethod; +import javassist.LoaderClassPath; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Inspired by: + * https://github.com/apache/dubbo/blob/master/dubbo-common/src/main/java/org/apache/dubbo/common/compiler/support/JavassistCompiler.java + */ +public class JavaReplCompiler { + + private static final Logger logger = LoggerFactory.getLogger(JavaReplCompiler.class); + + private static final Pattern IMPORT_PATTERN = Pattern.compile("import\\s+([\\w\\.\\*]+);\n"); + + private static final Pattern EXTENDS_PATTERN = + Pattern.compile("\\s+extends\\s+([\\w\\.]+)[^\\{]*\\{\n"); + + private static final Pattern IMPLEMENTS_PATTERN = + Pattern.compile("\\s+implements\\s+([\\w\\.]+)\\s*\\{\n"); + + private static final Pattern METHODS_PATTERN = + Pattern.compile("\n(private|public|protected)\\s+"); + + private static final Pattern FIELD_PATTERN = Pattern.compile("[^\n]+=[^\n]+;"); + + private static final Pattern PACKAGE_PATTERN = + Pattern.compile("package\\s+([$_a-zA-Z][$_a-zA-Z0-9\\.]*);"); + + private static final Pattern CLASS_PATTERN = + Pattern.compile("class\\s+([$_a-zA-Z][$_a-zA-Z0-9]*)\\s+"); + + private static final String javaReplClassName = "LinkisJavaRepl"; + + public void compileAndExecutor( + String code, ClassLoader classLoader, String classpathDir, String methodName) + throws Exception { + code = code.trim(); + Matcher matcher = PACKAGE_PATTERN.matcher(code); + String pkg; + if (matcher.find()) { + pkg = matcher.group(1); + } else { + pkg = ""; + } + matcher = CLASS_PATTERN.matcher(code); + Boolean containClass = true; + String cls; + if (matcher.find()) { + cls = matcher.group(1); + } else { + cls = javaReplClassName; + containClass = false; + } + String className = pkg != null && pkg.length() > 0 ? pkg + "." + cls : cls; + + // Check whether className exists. If so, change the name + try { + Class.forName(className, true, classLoader); + className = className.concat(RandomStringUtils.randomAlphabetic(6)); + } catch (ClassNotFoundException e) { + } + + logger.info("Java repl start building the class, className: {}", className); + + if (!code.endsWith("}")) { + throw new IllegalStateException("The java code not endsWith \"}\", code: \n" + code + "\n"); + } + + doCompileAndExecutor(classLoader, className, code, containClass, classpathDir, methodName); + } + + public void doCompileAndExecutor( + ClassLoader classLoader, + String name, + String source, + Boolean containClass, + String classpathDir, + String methodName) + throws Exception { + JavaReplBuilder builder = new JavaReplBuilder(); + builder.setClassName(name); + + // process imported classes + Matcher matcher = IMPORT_PATTERN.matcher(source); + while (matcher.find()) { + builder.addImports(matcher.group(1).trim()); + String importCode = matcher.group(); + if (StringUtils.isNotBlank(importCode)) { + source = source.replaceFirst(importCode, ""); + } + } + + // process extended super class + matcher = EXTENDS_PATTERN.matcher(source); + if (matcher.find()) { + builder.setSuperClassName(matcher.group(1).trim()); + } + + // process implemented interfaces + matcher = IMPLEMENTS_PATTERN.matcher(source); + if (matcher.find()) { + String[] ifaces = matcher.group(1).trim().split("\\,"); + Arrays.stream(ifaces).forEach(i -> builder.addInterface(i.trim())); + } + + // process constructors, fields, methods + String body = ""; + if (containClass) { + body = source.substring(source.indexOf('{') + 1, source.length() - 1); + } else { + body = source; + } + String[] methods = METHODS_PATTERN.split(body); + String className = getSimpleClassName(name); + Arrays.stream(methods) + .map(String::trim) + .filter(m -> !m.isEmpty()) + .forEach( + method -> { + if (method.startsWith(className)) { + builder.addConstructor("public " + method); + } else if (FIELD_PATTERN.matcher(method).matches()) { + builder.addField("private " + method); + } else { + builder.addMethod("public " + method); + } + }); + // compile + CtClass cls = builder.build(classLoader); + logger.info("Java repl CtClass build completed, CtClass: {}", cls); + + ClassPool cp = cls.getClassPool(); + if (classLoader == null) { + classLoader = cp.getClassLoader(); + } + cp.insertClassPath(new LoaderClassPath(classLoader)); + if (StringUtils.isNotBlank(classpathDir) && FileUtils.isDirectory(new File(classpathDir))) { + cp.insertClassPath(classpathDir); + } + + // If methodName is empty, get the first method + if (StringUtils.isBlank(methodName)) { + CtMethod[] declaredMethods = cls.getDeclaredMethods(); + String declareMethodName = declaredMethods[0].getName(); + if (ArrayUtils.isEmpty(declaredMethods) || StringUtils.isBlank(declareMethodName)) { + throw new ReplException( + ReplErrorCodeSummary.UNABLE_RESOLVE_JAVA_METHOD_NAME.getErrorCode(), + ReplErrorCodeSummary.UNABLE_RESOLVE_JAVA_METHOD_NAME.getErrorDesc()); + } + methodName = declareMethodName; + } + + logger.info("Java repl methodName: {}", methodName); + + logger.info("Java repl {} start executor", className); + + Class clazz = cp.toClass(cls); + Object obj = clazz.newInstance(); + obj.getClass().getMethod(methodName).invoke(obj); + logger.info("Java repl {} executor success", className); + } + + /** get simple class name from qualified class name */ + public static String getSimpleClassName(String qualifiedName) { + if (null == qualifiedName) { + return null; + } + int i = qualifiedName.lastIndexOf('.'); + return i < 0 ? qualifiedName : qualifiedName.substring(i + 1); + } +} diff --git a/linkis-engineconn-plugins/repl/src/main/resources/linkis-engineconn.properties b/linkis-engineconn-plugins/repl/src/main/resources/linkis-engineconn.properties new file mode 100644 index 0000000000..a6a89c7f04 --- /dev/null +++ b/linkis-engineconn-plugins/repl/src/main/resources/linkis-engineconn.properties @@ -0,0 +1,23 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +wds.linkis.server.version=v1 +#wds.linkis.engineconn.debug.enable=true +#wds.linkis.keytab.enable=true +wds.linkis.engineconn.plugin.default.class=org.apache.linkis.engineplugin.repl.ReplEngineConnPlugin + +wds.linkis.engineconn.support.parallelism=true + +wds.linkis.engineconn.max.free.time=0 \ No newline at end of file diff --git a/linkis-engineconn-plugins/repl/src/main/resources/log4j2.xml b/linkis-engineconn-plugins/repl/src/main/resources/log4j2.xml new file mode 100644 index 0000000000..2cd3e264c3 --- /dev/null +++ b/linkis-engineconn-plugins/repl/src/main/resources/log4j2.xml @@ -0,0 +1,91 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/linkis-engineconn-plugins/repl/src/main/resources/repl-ec.md b/linkis-engineconn-plugins/repl/src/main/resources/repl-ec.md new file mode 100644 index 0000000000..2f41238266 --- /dev/null +++ b/linkis-engineconn-plugins/repl/src/main/resources/repl-ec.md @@ -0,0 +1,117 @@ + +### 1. Submitting java tasks with Restful API +```text +POST /api/rest_j/v1/entrance/submit +``` + +#### 1.1. Contains the class name and method + +```java +package com.linkis.javassist; + +import org.apache.commons.lang3.StringUtils; + +/** + * @Author linkis + * @Date 2023/10/9 + */ +public class Test { + public void sayHello() { + System.out.println("hello"); + System.out.println(StringUtils.isEmpty("hello")); + } +} + +``` + +```json +{ + "executionContent":{ + "code":"package com.linkis.javassist;\n\nimport org.apache.commons.lang3.StringUtils;\n\n/**\n * @Author linkis\n * @Date 2023/10/9\n */\npublic class Test {\n public void sayHello() {\n System.out.println(\"hello\");\n System.out.println(StringUtils.isEmpty(\"hello\"));\n }\n}\n", + "runType":"repl" + }, + "params":{ + "configuration":{ + "runtime":{ + "linkis.repl.type":"java" + } + } + }, + "labels":{ + "engineType":"repl-1", + "userCreator":"linkis-IDE" + } +} +``` + +#### 1.2. Include method only + +```text +import org.apache.commons.lang3.StringUtils; + + public void sayHello() { + System.out.println("hello"); + System.out.println(StringUtils.isEmpty("hello")); + } +``` + +```json +{ + "executionContent":{ + "code":"import org.apache.commons.lang3.StringUtils;\n\n public void sayHello() {\n System.out.println(\"hello\");\n System.out.println(StringUtils.isEmpty(\"hello\"));\n }", + "runType":"repl" + }, + "params":{ + "configuration":{ + "runtime":{ + "linkis.repl.type":"java" + } + } + }, + "labels":{ + "engineType":"repl-1", + "userCreator":"linkis-IDE" + } +} +``` + +### 2. Submitting scala tasks with Restful API + +```text +import org.apache.commons.io.FileUtils +import java.io.File + +val x = 2 + 3; +println(x); +FileUtils.forceMkdir(new File("/tmp/linkis_repl_scala_test")); +``` + +```json +{ + "executionContent":{ + "code":"import org.apache.commons.io.FileUtils\nimport java.io.File\n\nval x = 2 + 3;\nprintln(x);\nFileUtils.forceMkdir(new File(\"/tmp/linkis_repl_scala_test\"));\n", + "runType":"repl" + }, + "params":{ + "configuration":{ + "runtime":{ + "linkis.repl.type":"scala" + } + } + }, + "labels":{ + "engineType":"repl-1", + "userCreator":"linkis-IDE" + } +} +``` + +### 3. Reference document +```text +http://www.javassist.org/tutorial/tutorial.html + +https://github.com/jboss-javassist/javassist + +https://github.com/apache/dubbo +``` + diff --git a/linkis-engineconn-plugins/repl/src/main/scala/org/apache/linkis/engineplugin/repl/factory/ReplEngineConnFactory.scala b/linkis-engineconn-plugins/repl/src/main/scala/org/apache/linkis/engineplugin/repl/factory/ReplEngineConnFactory.scala new file mode 100644 index 0000000000..fccb925535 --- /dev/null +++ b/linkis-engineconn-plugins/repl/src/main/scala/org/apache/linkis/engineplugin/repl/factory/ReplEngineConnFactory.scala @@ -0,0 +1,44 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.engineplugin.repl.factory + +import org.apache.linkis.engineconn.common.creation.EngineCreationContext +import org.apache.linkis.engineconn.common.engineconn.EngineConn +import org.apache.linkis.engineconn.computation.executor.creation.ComputationSingleExecutorEngineConnFactory +import org.apache.linkis.engineconn.executor.entity.LabelExecutor +import org.apache.linkis.engineplugin.repl.conf.ReplConfiguration +import org.apache.linkis.engineplugin.repl.executor.ReplEngineConnExecutor +import org.apache.linkis.manager.label.entity.engine.{EngineType, RunType} +import org.apache.linkis.manager.label.entity.engine.EngineType.EngineType +import org.apache.linkis.manager.label.entity.engine.RunType.RunType + +class ReplEngineConnFactory extends ComputationSingleExecutorEngineConnFactory { + + override def newExecutor( + id: Int, + engineCreationContext: EngineCreationContext, + engineConn: EngineConn + ): LabelExecutor = { + new ReplEngineConnExecutor(ReplConfiguration.ENGINE_DEFAULT_LIMIT.getValue, id) + } + + override protected def getEngineConnType: EngineType = EngineType.REPL + + override protected def getRunType: RunType = RunType.REPL + +} diff --git a/pom.xml b/pom.xml index e4d07644cf..5b310a8773 100644 --- a/pom.xml +++ b/pom.xml @@ -132,12 +132,16 @@ 1 0.234 3.0.0 + 1 python2 2.1.2 1 1.4.6 7.6.2 + + 3.28.0-GA + 5.4.1