Skip to content

Commit

Permalink
Merge branch 'master' into master-storage-parquet
Browse files Browse the repository at this point in the history
# Conflicts:
#	tool/dependencies/known-dependencies.txt
  • Loading branch information
ChengJie1053 committed Jan 17, 2024
2 parents ae439a1 + e79b59d commit 4b1870f
Show file tree
Hide file tree
Showing 88 changed files with 2,892 additions and 79 deletions.
2 changes: 1 addition & 1 deletion .github/workflows/integration-test.yml
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,7 @@ jobs:
TAG: ${{ github.sha }}
SKIP_TEST: true
HUB: ghcr.io/apache/linkis
LINKIS_VERSION: 1.5.0-SNAPSHOT
LINKIS_VERSION: 1.5.0
steps:
- name: Free up disk space
run: |
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/publish-docker.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@ jobs:
TAG: ${{ github.sha }}
SKIP_TEST: true
HUB: ghcr.io/apache/linkis
LINKIS_VERSION: 1.5.0-SNAPSHOT
LINKIS_VERSION: 1.5.0
steps:
- name: Checkout
uses: actions/checkout@v2
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/publish-snapshot.yml
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@ jobs:
fail-fast: false
matrix:
branch:
- dev-1.4.0
- dev-1.5.0
steps:
- name: Checkout repository
uses: actions/checkout@v2
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -631,7 +631,7 @@ class TaskExecutionServiceImpl
if (null != task) {
sendToEntrance(task, ResponseTaskLog(logUpdateEvent.taskId, logUpdateEvent.log))
} else {
logger.error("Task cannot null! logupdateEvent: " + logUpdateEvent.taskId)
logger.warn("Task cannot null! logupdateEvent: " + logUpdateEvent.taskId)
}
} else if (null != lastTask) {
val executor = executorManager.getReportExecutor
Expand Down Expand Up @@ -733,7 +733,7 @@ class TaskExecutionServiceImpl
if (null != executor) {
executor.getTaskById(taskId)
} else {
logger.error(s"Executor of taskId : $taskId is not cached.")
logger.warn(s"Executor of taskId : $taskId is not cached.")
null
}
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -245,7 +245,7 @@ object EntranceConfiguration {
val ENABLE_HDFS_JVM_USER =
CommonVars[Boolean]("linkis.entrance.enable.hdfs.jvm.user", true).getValue

val ENTRANCE_FAILOVER_ENABLED = CommonVars("linkis.entrance.failover.enable", true).getValue
val ENTRANCE_FAILOVER_ENABLED = CommonVars("linkis.entrance.failover.enable", false).getValue

val ENTRANCE_FAILOVER_SCAN_INIT_TIME =
CommonVars("linkis.entrance.failover.scan.init.time", 3 * 1000).getValue
Expand All @@ -269,7 +269,7 @@ object EntranceConfiguration {
val ENTRANCE_SHUTDOWN_FAILOVER_CONSUME_QUEUE_ENABLED =
CommonVars("linkis.entrance.shutdown.failover.consume.queue.enable", true).getValue

val ENTRANCE_GROUP_SCAN_ENABLED = CommonVars("linkis.entrance.group.scan.enable", true)
val ENTRANCE_GROUP_SCAN_ENABLED = CommonVars("linkis.entrance.group.scan.enable", false)

val ENTRANCE_GROUP_SCAN_INIT_TIME = CommonVars("linkis.entrance.group.scan.init.time", 3 * 1000)

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -69,8 +69,38 @@ trait CommentHelper {
object SQLCommentHelper extends CommentHelper {
override val commentPattern: Regex = """\s*--.+\s*""".r.unanchored
private val comment = "(?ms)('(?:''|[^'])*')|--.*?$|/\\*.*?\\*/|#.*?$|"
private val comment_sem = "(?i)(comment)\\s+'([^']*)'"
private val logger: Logger = LoggerFactory.getLogger(getClass)

def replaceComment(code: String): String = {
try {
val pattern = Pattern.compile(comment_sem)
val matcher = pattern.matcher(code)
val sb = new StringBuffer
while (matcher.find()) {
val commentKeyword = matcher.group(1)
val comment = matcher.group(2)

/**
* Since we are in a Scala string, and each backslash needs to be escaped in the string
* itself, we need two additional backslashes. Therefore, we end up with a total of four
* backslashes to represent a single literal backslash in the replacement string.
*/
val escapedComment = comment.replaceAll(";", "\\\\\\\\;")
matcher.appendReplacement(sb, commentKeyword + " '" + escapedComment + "'")
}
matcher.appendTail(sb)
sb.toString
} catch {
case e: Exception =>
logger.warn("sql comment semicolon replace failed")
code
case t: Throwable =>
logger.warn("sql comment semicolon replace failed")
code
}
}

override def dealComment(code: String): String = {
try {
val p = Pattern.compile(comment)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -121,7 +121,8 @@ object SQLExplain extends Explain {
logAppender: java.lang.StringBuilder
): Unit = {
val fixedCode: ArrayBuffer[String] = new ArrayBuffer[String]()
val tempCode = SQLCommentHelper.dealComment(executionCode)
val tempCode1 = SQLCommentHelper.dealComment(executionCode)
val tempCode = SQLCommentHelper.replaceComment(tempCode1)
val isNoLimitAllowed = Utils.tryCatch {
IDE_ALLOW_NO_LIMIT_REGEX.findFirstIn(executionCode).isDefined
} { case e: Exception =>
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,42 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package org.apache.linkis.entrance.interceptor.impl

import org.junit.jupiter.api.{Assertions, Test}

class TestReplaceComment {

@Test
def TestRepComm: Unit = {
val realCode = "drop table if exists default.test;" +
"create table default.test(" +
"id varchar(11) comment '这是注释测试分号;这是注释测试分号;'," +
"id1 string comment '测试'," +
"id2 string COMMENT '码值说明:2-高;3-中;4-低;'" +
");"
val expectCode = "drop table if exists default.test;" +
"create table default.test(" +
"id varchar(11) comment '这是注释测试分号\\;这是注释测试分号\\;'," +
"id1 string comment '测试'," +
"id2 string COMMENT '码值说明:2-高\\;3-中\\;4-低\\;'" +
");"

Assertions.assertEquals(SQLCommentHelper.replaceComment(realCode), expectCode)
}

}
Original file line number Diff line number Diff line change
Expand Up @@ -451,6 +451,7 @@ class LinkisSQLConnection(private[jdbc] val ujesClient: UJESClient, props: Prope
case EngineType.SPARK => RunType.SQL
case EngineType.HIVE => RunType.HIVE
case EngineType.REPL => RunType.REPL
case EngineType.DORIS => RunType.DORIS
case EngineType.TRINO => RunType.TRINO_SQL
case EngineType.PRESTO => RunType.PRESTO_SQL
case EngineType.NEBULA => RunType.NEBULA_SQL
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -244,6 +244,8 @@ class UJESSQLResultSet(
case null => throw new LinkisSQLException(LinkisSQLErrorCode.METADATA_EMPTY)
case "char" | "varchar" | "nvarchar" | "string" => value
case "short" => value.toShort
case "smallint" => value.toShort
case "tinyint" => value.toShort
case "int" => value.toInt
case "long" => value.toLong
case "float" => value.toFloat
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -69,7 +69,7 @@ public class AMConfiguration {
public static final CommonVars<String> MULTI_USER_ENGINE_TYPES =
CommonVars.apply(
"wds.linkis.multi.user.engine.types",
"jdbc,es,presto,io_file,appconn,openlookeng,trino,nebula,hbase");
"jdbc,es,presto,io_file,appconn,openlookeng,trino,nebula,hbase,doris");

public static final CommonVars<String> ALLOW_BATCH_KILL_ENGINE_TYPES =
CommonVars.apply("wds.linkis.allow.batch.kill.engine.types", "spark,hive,python");
Expand Down Expand Up @@ -105,8 +105,8 @@ public class AMConfiguration {
public static String getDefaultMultiEngineUser() {
String jvmUser = Utils.getJvmUser();
return String.format(
"{jdbc:\"%s\", es: \"%s\", presto:\"%s\", appconn:\"%s\", openlookeng:\"%s\", trino:\"%s\", nebula:\"%s\", hbase:\"%s\",io_file:\"root\"}",
jvmUser, jvmUser, jvmUser, jvmUser, jvmUser, jvmUser, jvmUser, jvmUser);
"{jdbc:\"%s\", es: \"%s\", presto:\"%s\", appconn:\"%s\", openlookeng:\"%s\", trino:\"%s\", nebula:\"%s\",doris:\"%s\", hbase:\"%s\",io_file:\"root\"}",
jvmUser, jvmUser, jvmUser, jvmUser, jvmUser, jvmUser, jvmUser, jvmUser, jvmUser);
}

public static boolean isMultiUserEngine(String engineType) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -237,8 +237,7 @@ public Message queryEcList(HttpServletRequest req, @RequestBody JsonNode jsonNod
// check special admin token
if (StringUtils.isNotBlank(token)) {
if (!Configuration.isAdminToken(token)) {
logger.warn("Token:{} has no permission to query ecList.", token);
return Message.error("Token:" + token + " has no permission to query ecList.");
return Message.error("Token has no permission to query ecList.");
}
} else if (!Configuration.isAdmin(username)) {
logger.warn("User:{} has no permission to query ecList.", username);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -479,8 +479,7 @@ public Message killEngineAsyn(HttpServletRequest req, @RequestBody JsonNode json
// check special token
if (StringUtils.isNotBlank(token)) {
if (!Configuration.isAdminToken(token)) {
logger.warn("Token {} has no permission to asyn kill engines.", token);
return Message.error("Token:" + token + " has no permission to asyn kill engines.");
return Message.error("Token has no permission to asyn kill engines.");
}
} else if (!Configuration.isAdmin(username)) {
logger.warn("User {} has no permission to asyn kill engines.", username);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -119,14 +119,11 @@ public static Resource getUserConfiguredResource(
UserCreatorLabel userCreatorLabel,
EngineTypeLabel engineTypeLabel) {
try {
Map<UserCreatorLabel, EngineTypeLabel> labelTuple =
new HashMap<UserCreatorLabel, EngineTypeLabel>();
labelTuple.put(userCreatorLabel, engineTypeLabel);
Resource userCreatorAvailableResource =
generateResource(
resourceType,
engineMapCache.getCacheMap(
buildRequestLabel(
userCreatorLabel.getUser(),
userCreatorLabel.getCreator(),
engineTypeLabel.getEngineType())));
generateResource(resourceType, engineMapCache.getCacheMap(labelTuple));
logger.info(
userCreatorLabel.getUser()
+ "on creator "
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -75,6 +75,9 @@ public class LabelCommonConfig {
public static final CommonVars<String> NEBULA_ENGINE_VERSION =
CommonVars.apply("wds.linkis.nebula.engine.version", "3.0.0");

public static final CommonVars<String> DORIS_ENGINE_VERSION =
CommonVars.apply("linkis.doris.engine.version", "1.2.6");

public static final CommonVars<String> PRESTO_ENGINE_VERSION =
CommonVars.apply("wds.linkis.presto.engine.version", "0.234");

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,8 @@ object EngineType extends Enumeration with Logging {

val REPL = Value("repl")

val DORIS = Value("doris")

val SHELL = Value("shell")

val JDBC = Value("jdbc")
Expand Down Expand Up @@ -98,6 +100,7 @@ object EngineType extends Enumeration with Logging {
case _ if PRESTO.toString.equalsIgnoreCase(str) => PRESTO
case _ if NEBULA.toString.equalsIgnoreCase(str) => NEBULA
case _ if REPL.toString.equalsIgnoreCase(str) => REPL
case _ if DORIS.toString.equalsIgnoreCase(str) => DORIS
case _ if FLINK.toString.equalsIgnoreCase(str) => FLINK
case _ if APPCONN.toString.equals(str) => APPCONN
case _ if SQOOP.toString.equalsIgnoreCase(str) => SQOOP
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@ object RunType extends Enumeration {
val SCALA = Value("scala")
val PYTHON = Value("python")
val REPL = Value("repl")
val DORIS = Value("doris")
val JAVA = Value("java")
val PYSPARK = Value("py")
val R = Value("r")
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -76,6 +76,8 @@ private static void init() {
EngineType.HBASE().toString(), LabelCommonConfig.HBASE_ENGINE_VERSION.getValue());
defaultVersion.put(
EngineType.NEBULA().toString(), LabelCommonConfig.NEBULA_ENGINE_VERSION.getValue());
defaultVersion.put(
EngineType.DORIS().toString(), LabelCommonConfig.DORIS_ENGINE_VERSION.getValue());
defaultVersion.put(
EngineType.SQOOP().toString(), LabelCommonConfig.SQOOP_ENGINE_VERSION.getValue());
defaultVersion.put(
Expand Down
5 changes: 3 additions & 2 deletions linkis-dist/bin/install.sh
Original file line number Diff line number Diff line change
Expand Up @@ -111,12 +111,13 @@ else
fi

if [ "$DISCOVERY" == "NACOS" ]; then
rm -rf $LINKIS_HOME/conf/application-*
mv $LINKIS_HOME/conf/nacos/* $LINKIS_HOME/conf
echo "Use NACOS mode"
else
mv $LINKIS_HOME/conf/eureka/* $LINKIS_HOME/conf
echo "Use EUREKA mode"
fi
rm -rf $LINKIS_HOME/conf/nacos
rm -rf $LINKIS_HOME/conf/eureka

cp ${LINKIS_CONFIG_PATH} $LINKIS_HOME/conf
cp ${LINKIS_DB_CONFIG_PATH} $LINKIS_HOME/conf
Expand Down
2 changes: 1 addition & 1 deletion linkis-dist/deploy-config/linkis-env.sh
Original file line number Diff line number Diff line change
Expand Up @@ -167,7 +167,7 @@ export SERVER_HEAP_SIZE="512M"
##The extended lib such mysql-connector-java-*.jar
#LINKIS_EXTENDED_LIB=/appcom/common/linkisExtendedLib

LINKIS_VERSION=1.4.0
LINKIS_VERSION=1.5.0

# for install
LINKIS_PUBLIC_MODULE=lib/linkis-commons/public-module
Expand Down
23 changes: 21 additions & 2 deletions linkis-dist/package/db/linkis_ddl.sql
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,7 @@ CREATE TABLE `linkis_ps_configuration_config_key`(
`en_description` varchar(200) DEFAULT NULL COMMENT 'english description',
`en_name` varchar(100) DEFAULT NULL COMMENT 'english name',
`en_treeName` varchar(100) DEFAULT NULL COMMENT 'english treeName',
`template_required` tinyint(1) DEFAULT 0 COMMENT 'template required 0 none / 1 must',
PRIMARY KEY (`id`)
)ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_bin;

Expand All @@ -64,7 +65,7 @@ DROP TABLE IF EXISTS `linkis_ps_configuration_config_value`;
CREATE TABLE `linkis_ps_configuration_config_value`(
`id` bigint(20) NOT NULL AUTO_INCREMENT,
`config_key_id` bigint(20),
`config_value` varchar(200),
`config_value` varchar(500),
`config_label_id`int(20),
`update_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP,
`create_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP,
Expand All @@ -85,7 +86,25 @@ CREATE TABLE `linkis_ps_configuration_category` (
UNIQUE INDEX `uniq_label_id` (`label_id`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_bin;


DROP TABLE IF EXISTS `linkis_ps_configuration_template_config_key`;
CREATE TABLE IF NOT EXISTS `linkis_ps_configuration_template_config_key` (
`id` BIGINT(20) NOT NULL AUTO_INCREMENT,
`template_name` VARCHAR(200) NOT NULL COMMENT 'Configuration template name redundant storage',
`template_uuid` VARCHAR(36) NOT NULL COMMENT 'uuid template id recorded by the third party',
`key_id` BIGINT(20) NOT NULL COMMENT 'id of linkis_ps_configuration_config_key',
`config_value` VARCHAR(200) NULL DEFAULT NULL COMMENT 'configuration value',
`max_value` VARCHAR(50) NULL DEFAULT NULL COMMENT 'upper limit value',
`min_value` VARCHAR(50) NULL DEFAULT NULL COMMENT 'Lower limit value (reserved)',
`validate_range` VARCHAR(50) NULL DEFAULT NULL COMMENT 'Verification regularity (reserved)',
`is_valid` VARCHAR(2) DEFAULT 'Y' COMMENT 'Is it valid? Reserved Y/N',
`create_by` VARCHAR(50) NOT NULL COMMENT 'Creator',
`create_time` DATETIME DEFAULT CURRENT_TIMESTAMP COMMENT 'create time',
`update_by` VARCHAR(50) NULL DEFAULT NULL COMMENT 'Update by',
`update_time` DATETIME DEFAULT CURRENT_TIMESTAMP COMMENT 'update time',
PRIMARY KEY (`id`),
UNIQUE INDEX `uniq_tid_kid` (`template_uuid`, `key_id`),
UNIQUE INDEX `uniq_tname_kid` (`template_uuid`, `key_id`)
)ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin;

DROP TABLE IF EXISTS `linkis_ps_configuration_key_limit_for_user`;
CREATE TABLE IF NOT EXISTS `linkis_ps_configuration_key_limit_for_user` (
Expand Down
8 changes: 4 additions & 4 deletions linkis-dist/package/db/linkis_ddl_pg.sql
Original file line number Diff line number Diff line change
Expand Up @@ -40,12 +40,12 @@ CREATE TABLE linkis_ps_configuration_config_key (
is_hidden bool NULL,
is_advanced bool NULL,
"level" int2 NULL,
boundary_type int2 null,
"treeName" varchar(20) NULL,
"treeName" varchar(20) NULL,
boundary_type int2 NOT NULL DEFAULT 0,
en_description varchar(200) NULL,
en_name varchar(100) NULL,
"en_treeName" varchar(100) NULL,
`boundary_type` int2 NOT NULL,
template_required bool NULL,
CONSTRAINT linkis_configuration_config_key_pkey PRIMARY KEY (id)
);
COMMENT ON COLUMN "linkis_ps_configuration_config_key"."key" IS 'Set key, e.g. spark.executor.instances';
Expand Down Expand Up @@ -136,7 +136,7 @@ CREATE TABLE IF NOT EXISTS linkis_ps_configuration_key_limit_for_user (

DROP TABLE IF EXISTS linkis_ps_configutation_lm_across_cluster_rule;
CREATE TABLE IF NOT EXISTS linkis_ps_configutation_lm_across_cluster_rule (
rule_id INT PRIMARY KEY AUTOINCREMENT,
rule_id bigserial NOT NULL,
cluster_name varchar(32) NOT NULL,
creator varchar(32) NOT NULL,
username varchar(32) NOT NULL,
Expand Down
3 changes: 3 additions & 0 deletions linkis-dist/package/db/linkis_dml.sql
Original file line number Diff line number Diff line change
Expand Up @@ -777,3 +777,6 @@ VALUES (@data_source_type_id, 'address', '地址', 'Address', NULL, 'TEXT', NULL
(@data_source_type_id, 'username', '用户名(Username)', 'Username', NULL, 'TEXT', NULL, 1, '用户名(Username)', 'Username', '^[0-9A-Za-z_-]+$', NULL, NULL, NULL, now(), now()),
(@data_source_type_id, 'password', '密码(Password)', 'Password', NULL, 'PASSWORD', NULL, 1, '密码(Password)', 'Password', '', NULL, NULL, NULL, now(), now()),
(@data_source_type_id, 'instance', '实例名(instance)', 'Instance', NULL, 'TEXT', NULL, 1, '实例名(instance)', 'Instance', NULL, NULL, NULL, NULL, now(), now());

select @data_source_type_id := id from `linkis_ps_dm_datasource_type` where `name` = 'doris';
UPDATE linkis_ps_dm_datasource_type_key SET `require` = 0 WHERE `key` ="password" and `data_source_type_id` = @data_source_type_id;
2 changes: 1 addition & 1 deletion linkis-dist/package/db/linkis_dml_pg.sql
Original file line number Diff line number Diff line change
Expand Up @@ -647,7 +647,7 @@ VALUES ((select id from "linkis_ps_dm_datasource_type" where "name" = 'doris'),
((select id from "linkis_ps_dm_datasource_type" where "name" = 'doris'), 'driverClassName', '驱动类名(Driver class name)', 'Driver class name', 'com.mysql.jdbc.Driver', 'TEXT', NULL, '1', '驱动类名(Driver class name)', 'Driver class name', NULL, NULL, NULL, NULL, now(), now()),
((select id from "linkis_ps_dm_datasource_type" where "name" = 'doris'), 'params', '连接参数(Connection params)', 'Connection params', NULL, 'TEXT', NULL, '0', '输入JSON格式(Input JSON format): {"param":"value"}', 'Input JSON format: {"param":"value"}', NULL, NULL, NULL, NULL, now(), now()),
((select id from "linkis_ps_dm_datasource_type" where "name" = 'doris'), 'username', '用户名(Username)', 'Username', NULL, 'TEXT', NULL, '1', '用户名(Username)', 'Username', '^[0-9A-Za-z_-]+$', NULL, NULL, NULL, now(), now()),
((select id from "linkis_ps_dm_datasource_type" where "name" = 'doris'), 'password', '密码(Password)', 'Password', NULL, 'PASSWORD', NULL, '1', '密码(Password)', 'Password', '', NULL, NULL, NULL, now(), now()),
((select id from "linkis_ps_dm_datasource_type" where "name" = 'doris'), 'password', '密码(Password)', 'Password', NULL, 'PASSWORD', NULL, '0', '密码(Password)', 'Password', '', NULL, NULL, NULL, now(), now()),
((select id from "linkis_ps_dm_datasource_type" where "name" = 'doris'), 'instance', '实例名(instance)', 'Instance', NULL, 'TEXT', NULL, '1', '实例名(instance)', 'Instance', NULL, NULL, NULL, NULL, now(), now());

-- https://github.com/ClickHouse/clickhouse-jdbc/tree/master/clickhouse-jdbc
Expand Down
Loading

0 comments on commit 4b1870f

Please sign in to comment.