Skip to content

Commit

Permalink
[Feature] Add hbase engine to linkis
Browse files Browse the repository at this point in the history
  • Loading branch information
Longping Jie committed Aug 31, 2023
1 parent faa8d15 commit c452042
Show file tree
Hide file tree
Showing 222 changed files with 17,435 additions and 3 deletions.
7 changes: 4 additions & 3 deletions ...-application-manager/src/main/java/org/apache/linkis/manager/am/conf/AMConfiguration.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -68,7 +68,8 @@ public class AMConfiguration {

public static final CommonVars<String> MULTI_USER_ENGINE_TYPES =
CommonVars.apply(
"wds.linkis.multi.user.engine.types", "jdbc,es,presto,io_file,appconn,openlookeng,trino");
"wds.linkis.multi.user.engine.types",
"jdbc,es,presto,io_file,appconn,openlookeng,trino,hbase");

public static final CommonVars<String> ALLOW_BATCH_KILL_ENGINE_TYPES =
CommonVars.apply("wds.linkis.allow.batch.kill.engine.types", "spark,hive,python");
Expand Down Expand Up @@ -104,8 +105,8 @@ public class AMConfiguration {
public static String getDefaultMultiEngineUser() {
String jvmUser = Utils.getJvmUser();
return String.format(
"{jdbc:\"%s\", es: \"%s\", presto:\"%s\", appconn:\"%s\", openlookeng:\"%s\", trino:\"%s\", io_file:\"root\"}",
jvmUser, jvmUser, jvmUser, jvmUser, jvmUser, jvmUser);
"{jdbc:\"%s\", es: \"%s\", presto:\"%s\", appconn:\"%s\", openlookeng:\"%s\", trino:\"%s\", hbase:\"%s\", io_file:\"root\"}",
jvmUser, jvmUser, jvmUser, jvmUser, jvmUser, jvmUser, jvmUser);
}

public static boolean isMultiUserEngine(String engineType) {
Expand Down
3 changes: 3 additions & 0 deletions ...is-label-common/src/main/java/org/apache/linkis/manager/label/conf/LabelCommonConfig.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -72,6 +72,9 @@ public class LabelCommonConfig {
public static final CommonVars<String> PRESTO_ENGINE_VERSION =
CommonVars.apply("wds.linkis.presto.engine.version", "0.234");

public static final CommonVars<String> HBASE_ENGINE_VERSION =
CommonVars.apply("wds.linkis.hbase.engine.version", "2.5.3");

public static final CommonVars<String> OPENLOOKENG_ENGINE_VERSION =
CommonVars.apply("wds.linkis.openlookeng.engine.version", "1.5.0");

Expand Down
3 changes: 3 additions & 0 deletions ...label-common/src/main/java/org/apache/linkis/manager/label/entity/engine/EngineType.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,8 @@ object EngineType extends Enumeration with Logging {

val HIVE = Value("hive")

val HBASE = Value("hbase")

val PYTHON = Value("python")

val SHELL = Value("shell")
Expand Down Expand Up @@ -82,6 +84,7 @@ object EngineType extends Enumeration with Logging {
case _ if null == str || "".equals(str) => null
case _ if SPARK.toString.equalsIgnoreCase(str) => SPARK
case _ if HIVE.toString.equalsIgnoreCase(str) => HIVE
case _ if HBASE.toString.equalsIgnoreCase(str) => HBASE
case _ if PYTHON.toString.equalsIgnoreCase(str) => PYTHON
case _ if SHELL.toString.equalsIgnoreCase(str) => SHELL
case _ if JDBC.toString.equalsIgnoreCase(str) => JDBC
Expand Down
2 changes: 2 additions & 0 deletions ...el-common/src/main/java/org/apache/linkis/manager/label/utils/EngineTypeLabelCreator.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -69,6 +69,8 @@ private static void init() {
EngineType.FLINK().toString(), LabelCommonConfig.FLINK_ENGINE_VERSION.getValue());
defaultVersion.put(
EngineType.PRESTO().toString(), LabelCommonConfig.PRESTO_ENGINE_VERSION.getValue());
defaultVersion.put(
EngineType.HBASE().toString(), LabelCommonConfig.HBASE_ENGINE_VERSION.getValue());
defaultVersion.put(
EngineType.SQOOP().toString(), LabelCommonConfig.SQOOP_ENGINE_VERSION.getValue());
defaultVersion.put(
Expand Down
33 changes: 33 additions & 0 deletions linkis-dist/package/db/upgrade/1.5.0_schema/mysql/linkis_dml.sql
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,33 @@
SET @HBASE_LABEL="hbase-2.5.3";
SET @HBASE_ALL=CONCAT('*-*,',@HBASE_LABEL);
SET @HBASE_IDE=CONCAT('*-IDE,',@HBASE_LABEL);

-- hbase
INSERT INTO `linkis_ps_configuration_config_key` (`key`, `description`, `name`, `default_value`, `validate_type`, `validate_range`, `engine_conn_type`, `is_hidden`, `is_advanced`, `level`, `treeName`) VALUES ('wds.linkis.hbase.zookeeper.quorum', 'HBase 集群连接', 'zk连接地址', 'localhost', 'None', NULL, 'hbase', 0, 0, 1, '数据源配置');
INSERT INTO `linkis_ps_configuration_config_key` (`key`, `description`, `name`, `default_value`, `validate_type`, `validate_range`, `engine_conn_type`, `is_hidden`, `is_advanced`, `level`, `treeName`) VALUES ('wds.linkis.hbase.zookeeper.property.clientPort', 'HBase 集群连接', 'zk连接端口', '2181', 'None', NULL, 'hbase', 0, 0, 1, '数据源配置');
INSERT INTO `linkis_ps_configuration_config_key` (`key`, `description`, `name`, `default_value`, `validate_type`, `validate_range`, `engine_conn_type`, `is_hidden`, `is_advanced`, `level`, `treeName`) VALUES ('wds.linkis.zookeeper.znode.parent', 'HBase 集群连接', 'znodeParent', '/hbase', 'None', NULL, 'hbase', 0, 0, 1, '数据源配置');
INSERT INTO `linkis_ps_configuration_config_key` (`key`, `description`, `name`, `default_value`, `validate_type`, `validate_range`, `engine_conn_type`, `is_hidden`, `is_advanced`, `level`, `treeName`) VALUES ('wds.linkis.hbase.rootdir', 'HBase 集群连接', 'hbase root dir', '/hbase', 'None', NULL, 'hbase', 0, 0, 1, '数据源配置');
INSERT INTO `linkis_ps_configuration_config_key` (`key`, `description`, `name`, `default_value`, `validate_type`, `validate_range`, `engine_conn_type`, `is_hidden`, `is_advanced`, `level`, `treeName`) VALUES ('wds.linkis.hbase.security.authentication', 'HBase 集群连接', '认证方式', 'simple', 'OFT', '[\"simple\",\"kerberos\"]', 'hbase', 0, 0, 1, '数据源配置');
INSERT INTO `linkis_ps_configuration_config_key` (`key`, `description`, `name`, `default_value`, `validate_type`, `validate_range`, `engine_conn_type`, `is_hidden`, `is_advanced`, `level`, `treeName`) VALUES ('wds.linkis.hbase.kerberos.principal', 'HBase 集群连接', 'principal', '', 'None', NULL, 'hbase', 0, 0, 1, '数据源配置');
INSERT INTO `linkis_ps_configuration_config_key` (`key`, `description`, `name`, `default_value`, `validate_type`, `validate_range`, `engine_conn_type`, `is_hidden`, `is_advanced`, `level`, `treeName`) VALUES ('wds.linkis.hbase.keytab.file', 'HBase 集群连接', 'keytab', '', 'None', NULL, 'hbase', 0, 0, 1, '数据源配置');
INSERT INTO `linkis_ps_configuration_config_key` (`key`, `description`, `name`, `default_value`, `validate_type`, `validate_range`, `engine_conn_type`, `is_hidden`, `is_advanced`, `level`, `treeName`) VALUES ('wds.linkis.hbase.kerberos.proxy.user', 'HBase 集群连接', 'kerberos用户代理', '', 'None', NULL, 'hbase', 0, 0, 1, '数据源配置');
INSERT INTO `linkis_ps_configuration_config_key` (`key`, `description`, `name`, `default_value`, `validate_type`, `validate_range`, `engine_conn_type`, `is_hidden`, `is_advanced`, `level`, `treeName`) VALUES ('wds.linkis.hbase.regionserver.kerberos.principal', 'HBase 集群连接', 'region server principal', '', 'None', NULL, 'hbase', 0, 0, 1, '数据源配置');
INSERT INTO `linkis_ps_configuration_config_key` (`key`, `description`, `name`, `default_value`, `validate_type`, `validate_range`, `engine_conn_type`, `is_hidden`, `is_advanced`, `level`, `treeName`) VALUES ('wds.linkis.hbase.master.kerberos.principal', 'HBase 集群连接', 'master principal', '', 'None', NULL, 'hbase', 0, 0, 1, '数据源配置');


insert into `linkis_cg_manager_label` (`label_key`, `label_value`, `label_feature`, `label_value_size`, `update_time`, `create_time`) VALUES ('combined_userCreator_engineType', @HBASE_ALL, 'OPTIONAL', 2, now(), now());
insert into `linkis_cg_manager_label` (`label_key`, `label_value`, `label_feature`, `label_value_size`, `update_time`, `create_time`) VALUES ('combined_userCreator_engineType', @HBASE_IDE, 'OPTIONAL', 2, now(), now());


-- hbase-2.5.3
insert into `linkis_ps_configuration_key_engine_relation` (`config_key_id`, `engine_type_label_id`)
(select config.id as `config_key_id`, label.id AS `engine_type_label_id` FROM linkis_ps_configuration_config_key config
INNER JOIN linkis_cg_manager_label label ON config.engine_conn_type = 'hbase' and label_value = @HBASE_ALL);




-- hbase default configuration
insert into `linkis_ps_configuration_config_value` (`config_key_id`, `config_value`, `config_label_id`)
(select `relation`.`config_key_id` AS `config_key_id`, '' AS `config_value`, `relation`.`engine_type_label_id` AS `config_label_id` FROM linkis_ps_configuration_key_engine_relation relation
INNER JOIN linkis_cg_manager_label label ON relation.engine_type_label_id = label.id AND label.label_value = @HBASE_ALL);
259 changes: 259 additions & 0 deletions linkis-engineconn-plugins/hbase/pom.xml
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,259 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.apache.linkis</groupId>
<artifactId>linkis</artifactId>
<version>${revision}</version>
<relativePath>../../pom.xml</relativePath>
</parent>

<artifactId>linkis-engineplugin-hbase</artifactId>

<dependencies>

<dependency>
<groupId>org.apache.linkis</groupId>
<artifactId>linkis-engineconn-plugin-core</artifactId>
<version>${project.version}</version>
</dependency>

<dependency>
<groupId>org.apache.linkis</groupId>
<artifactId>linkis-computation-engineconn</artifactId>
<version>${project.version}</version>
</dependency>

<dependency>
<groupId>org.apache.linkis</groupId>
<artifactId>linkis-storage</artifactId>
<version>${project.version}</version>
<scope>provided</scope>
</dependency>

<dependency>
<groupId>org.apache.linkis</groupId>
<artifactId>linkis-common</artifactId>
<version>${project.version}</version>
<scope>provided</scope>
</dependency>

<dependency>
<groupId>org.apache.linkis</groupId>
<artifactId>linkis-rpc</artifactId>
<version>${project.version}</version>
<scope>provided</scope>
</dependency>

<!-- <dependency>
<groupId>org.apache.hbase</groupId>
<artifactId>hbase-server</artifactId>
<version>${hbase.version}</version>
<exclusions>
<exclusion>
<groupId>org.jruby.jcodings</groupId>
<artifactId>jcodings</artifactId>
</exclusion>
<exclusion>
<groupId>org.jruby.joni</groupId>
<artifactId>joni</artifactId>
</exclusion>
<exclusion>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-log4j12</artifactId>
</exclusion>
<exclusion>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-1.2-api</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-slf4j-impl</artifactId>
</exclusion>
<exclusion>
<groupId>org.eclipse.jetty</groupId>
<artifactId>jetty-util</artifactId>
</exclusion>
<exclusion>
<groupId>org.eclipse.jetty</groupId>
<artifactId>jetty-io</artifactId>
</exclusion>
<exclusion>
<groupId>org.eclipse.jetty</groupId>
<artifactId>jetty-client</artifactId>
</exclusion>
<exclusion>
<artifactId>hadoop-mapreduce-client-core</artifactId>
<groupId>org.apache.hadoop</groupId>
</exclusion>
<exclusion>
<artifactId>hbase-http</artifactId>
<groupId>org.apache.hbase</groupId>
</exclusion>
</exclusions>
</dependency>-->
<dependency>
<groupId>org.apache.hbase</groupId>
<artifactId>hbase-server</artifactId>
<exclusions>
<exclusion>
<groupId>commons-codec</groupId>
<artifactId>commons-codec</artifactId>
</exclusion>
<exclusion>
<groupId>commons-io</groupId>
<artifactId>commons-io</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.commons</groupId>
<artifactId>commons-lang3</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.commons</groupId>
<artifactId>commons-math3</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-auth</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-mapreduce-client-core</artifactId>
</exclusion>
<exclusion>
<groupId>javax.ws.rs</groupId>
<artifactId>jsr311-api</artifactId>
</exclusion>
<exclusion>
<groupId>org.mortbay.jetty</groupId>
<artifactId>jetty</artifactId>
</exclusion>
<exclusion>
<groupId>org.mortbay.jetty</groupId>
<artifactId>jetty-util</artifactId>
</exclusion>
<exclusion>
<groupId>javax.servlet</groupId>
<artifactId>javax.servlet-api</artifactId>
</exclusion>
<exclusion>
<groupId>org.glassfish.web</groupId>
<artifactId>javax.servlet.jsp</artifactId>
</exclusion>
<exclusion>
<groupId>javax.servlet.jsp</groupId>
<artifactId>javax.servlet.jsp-api</artifactId>
</exclusion>
</exclusions>
</dependency>

<dependency>
<groupId>org.jruby</groupId>
<artifactId>jruby-complete</artifactId>
<version>${jruby.verison}</version>
</dependency>
<dependency>
<groupId>org.jruby.jcodings</groupId>
<artifactId>jcodings</artifactId>
<version>${jcodings.version}</version>
</dependency>
<dependency>
<groupId>org.jruby.joni</groupId>
<artifactId>joni</artifactId>
<version>${joni.version}</version>
</dependency>

<dependency>
<groupId>org.apache.hbase</groupId>
<artifactId>hbase-testing-util</artifactId>
<version>${hbase.version}</version>
<scope>test</scope>
<exclusions>
<exclusion>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
</exclusion>
</exclusions>
</dependency>

<dependency>
<groupId>org.apache.httpcomponents</groupId>
<artifactId>httpclient</artifactId>
<version>${httpclient.version}</version>
</dependency>
<dependency>
<groupId>org.apache.httpcomponents</groupId>
<artifactId>httpmime</artifactId>
<version>${httpmime.version}</version>
</dependency>

<dependency>
<groupId>org.scala-lang</groupId>
<artifactId>scala-library</artifactId>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.scala-lang</groupId>
<artifactId>scala-compiler</artifactId>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.scala-lang</groupId>
<artifactId>scala-reflect</artifactId>
<scope>provided</scope>
</dependency>

<dependency>
<groupId>io.netty</groupId>
<artifactId>netty-all</artifactId>
<scope>provided</scope>
</dependency>
</dependencies>

<build>
<plugins>
<plugin>
<groupId>net.alchim31.maven</groupId>
<artifactId>scala-maven-plugin</artifactId>
</plugin>

<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-assembly-plugin</artifactId>
<inherited>false</inherited>
<configuration>
<skipAssembly>false</skipAssembly>
<finalName>out</finalName>
<appendAssemblyId>false</appendAssemblyId>
<attach>false</attach>
<descriptors>
<descriptor>src/main/assembly/distribution.xml</descriptor>
</descriptors>
</configuration>
<executions>
<execution>
<id>make-assembly</id>
<goals>
<goal>single</goal>
</goals>
<phase>package</phase>
<configuration>
<descriptors>
<descriptor>src/main/assembly/distribution.xml</descriptor>
</descriptors>
</configuration>
</execution>
</executions>
</plugin>
</plugins>
</build>

</project>
Loading

0 comments on commit c452042

Please sign in to comment.