diff --git a/linkis-dist/bin/install.sh b/linkis-dist/bin/install.sh index 9702e1e005..a80deea117 100644 --- a/linkis-dist/bin/install.sh +++ b/linkis-dist/bin/install.sh @@ -128,6 +128,7 @@ RANDOM_LINKISCLI_TOKEN="LINKISCLI-`cat /proc/sys/kernel/random/uuid | awk -F- '{ if [ $DEBUG_MODE != "true" ];then sed -i ${txt} "s#BML-AUTH#$RANDOM_BML_TOKEN#g" $LINKIS_HOME/conf/linkis-cli/linkis-cli.properties sed -i ${txt} "s#BML-AUTH#$RANDOM_BML_TOKEN#g" $common_conf + sed -i ${txt} "s#BML-AUTH#$RANDOM_BML_TOKEN#g" $LINKIS_HOME/admin/configuration_helper.sh sed -i ${txt} "s#LINKIS_CLI_TEST#$RANDOM_LINKIS_CLI_TEST_TOKEN#g" $common_conf sed -i ${txt} "s#WS-AUTH#$RANDOM_WS_TOKEN#g" $common_conf sed -i ${txt} "s#DSM-AUTH#$RANDOM_DSM_TOKEN#g" $common_conf @@ -533,6 +534,7 @@ sed -i ${txt} "s#spring.eureka.instance.metadata-map.linkis.conf.version.*#spri if [ "$RESULT_SET_ROOT_PATH" != "" ] then sed -i ${txt} "s#wds.linkis.resultSet.store.path.*#wds.linkis.resultSet.store.path=$RESULT_SET_ROOT_PATH#g" $entrance_conf + sed -i ${txt} "s#resultSetRootDir=.*#resultSetRootDir=$RESULT_SET_ROOT_PATH#g" $LINKIS_HOME/admin/linkis_task_res_log_clear.sh fi publicservice_conf=$LINKIS_HOME/conf/linkis-ps-publicservice.properties diff --git a/linkis-dist/package/admin/clear_ec_record.sh b/linkis-dist/package/admin/clear_ec_record.sh new file mode 100644 index 0000000000..5cd1525263 --- /dev/null +++ b/linkis-dist/package/admin/clear_ec_record.sh @@ -0,0 +1,51 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# +# description: clear linkis_ps_job_history_group_history 10 days ago record +# +if [ -f ${LINKIS_CONF_DIR}/db.sh ] +then + export LINKIS_DB_CONFIG_PATH=${LINKIS_CONF_DIR}/db.sh +else + if [ -f ${LINKIS_HOME}/conf/db.sh ] + then + export LINKIS_DB_CONFIG_PATH=${LINKIS_HOME}/conf/db.sh + else + echo "can not find db.sh" + exit + fi +fi +source ${LINKIS_DB_CONFIG_PATH} + +delete_day=`date -d "-10 days" "+%Y-%m-%d"` +delte_time="$delete_day 00:00:00" +echo "start to delete linkis_cg_ec_resource_info_record before $delte_time" +parm="release_time <=\"$delte_time\" " + +count=`mysql -h$MYSQL_HOST -P$MYSQL_PORT -u$MYSQL_USER -p$MYSQL_PASSWORD $MYSQL_DB -ss -e "SELECT count(1) FROM linkis_cg_ec_resource_info_record where $parm limit 1"` +maxid=`mysql -h$MYSQL_HOST -P$MYSQL_PORT -u$MYSQL_USER -p$MYSQL_PASSWORD $MYSQL_DB -ss -e "SELECT MAX(id) FROM linkis_cg_ec_resource_info_record where $parm limit 1"` +echo "will delete count:$count" +echo "maxid:$maxid" + +while [ $count -gt 1 ];do + mysql -h$MYSQL_HOST -P$MYSQL_PORT -u$MYSQL_USER -p$MYSQL_PASSWORD $MYSQL_DB -ss -e "DELETE FROM linkis_cg_ec_resource_info_record where id <= $maxid limit 5000;" + count=`mysql -h$MYSQL_HOST -P$MYSQL_PORT -u$MYSQL_USER -p$MYSQL_PASSWORD $MYSQL_DB -ss -e "SELECT count(1) FROM linkis_cg_ec_resource_info_record where $parm limit 1"` + echo "count change ： $count" + sleep 1s +done + +echo "clear_ec_record.sh over" \ No newline at end of file diff --git a/linkis-dist/package/admin/clear_history_task.sh b/linkis-dist/package/admin/clear_history_task.sh new file mode 100644 index 0000000000..75c49cb715 --- /dev/null +++ b/linkis-dist/package/admin/clear_history_task.sh @@ -0,0 +1,49 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# +# description: clear linkis_ps_job_history_group_history 3 month record +# +if [ -f ${LINKIS_CONF_DIR}/db.sh ] +then + export LINKIS_DB_CONFIG_PATH=${LINKIS_CONF_DIR}/db.sh +else + if [ -f ${LINKIS_HOME}/conf/db.sh ] + then + export LINKIS_DB_CONFIG_PATH=${LINKIS_HOME}/conf/db.sh + else + echo "can not find db.sh" + exit + fi +fi +source ${LINKIS_DB_CONFIG_PATH} + +delete_day=`date -d "-90 days" "+%Y-%m-%d"` +delte_time="$delete_day 00:00:00" +echo "start to delete linkis_ps_job_history_group_history before $delte_time" +parm="created_time <=\"$delte_time\" " + +count=`mysql -h$MYSQL_HOST -P$MYSQL_PORT -u$MYSQL_USER -p$MYSQL_PASSWORD $MYSQL_DB -ss -e "SELECT count(1) FROM linkis_ps_job_history_group_history where $parm limit 1 "` +maxid=`mysql -h$MYSQL_HOST -P$MYSQL_PORT -u$MYSQL_USER -p$MYSQL_PASSWORD $MYSQL_DB -ss -e "SELECT MAX(id) FROM linkis_ps_job_history_group_history where $parm limit 1 "` +echo "will delete count:$count" +echo "maxid:$maxid" + +while [ $count -gt 1 ];do + mysql -h$MYSQL_HOST -P$MYSQL_PORT -u$MYSQL_USER -p$MYSQL_PASSWORD $MYSQL_DB -ss -e "DELETE FROM linkis_ps_job_history_group_history where id <= $maxid limit 5000;" + count=`mysql -h$MYSQL_HOST -P$MYSQL_PORT -u$MYSQL_USER -p$MYSQL_PASSWORD $MYSQL_DB -ss -e "SELECT count(1) FROM linkis_ps_job_history_group_history where $parm limit 1 "` + echo "count change ： $count" + sleep 1s +done \ No newline at end of file diff --git a/linkis-dist/package/admin/configuration_helper.sh b/linkis-dist/package/admin/configuration_helper.sh new file mode 100644 index 0000000000..8c918dfe61 --- /dev/null +++ b/linkis-dist/package/admin/configuration_helper.sh @@ -0,0 +1,89 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + + +cd `dirname $0` +cd .. +INSTALL_HOME=`pwd` + + +function print_usage(){ + echo "Usage: configuration_helper.sh [add | get | delete] [engineType] [version] [creator] [configKey] [configValue option]" + echo "get eq: sh configuration_helper.sh get spark 2.4.3 test wds.linkis.rm.instance hadoop" + echo "delete eq: sh configuration_helper.sh delete spark 2.4.3 test wds.linkis.rm.instance hadoop" + echo "add eq: sh configuration_helper.sh add spark 2.4.3 test wds.linkis.rm.instance hadoop 6" + echo "add eq: sh configuration_helper.sh add spark 2.4.3 test wds.linkis.rm.instance hadoop 6 force" + echo "add tips: add with force will ignore check error" + echo "Most commands print help when invoked w/o parameters." +} + +if [ $# -lt 6 ]; then + print_usage + exit 2 +fi + +# set LINKIS_HOME +if [ "$LINKIS_HOME" = "" ]; then + export LINKIS_HOME=$INSTALL_HOME +fi + +# set LINKIS_CONF_DIR +if [ "$LINKIS_CONF_DIR" = "" ]; then + export LINKIS_CONF_DIR=$LINKIS_HOME/conf +fi +linkisMainConf=$LINKIS_CONF_DIR/linkis.properties +gatewayUrl=$(grep wds.linkis.gateway.url $linkisMainConf | cut -d"=" -f2) +echo "gatewayUrl: $gatewayUrl" +engineType=$2 +version=$3 +creator=$4 +configKey=$5 +user=$6 +configValue=$7 +COMMAND=$1 +if [ "$8" = "force" ]; then + force=true +fi + +get() +{ + requestUrl="$gatewayUrl/api/rest_j/v1/configuration/keyvalue?creator=$creator&engineType=$engineType&version=$version&configKey=$configKey" + curl --location --request GET $requestUrl -H "Token-Code:BML-AUTH" -H "Token-User:$user" +} + +delete() +{ + requestUrl="$gatewayUrl/api/rest_j/v1/configuration/keyvalue" + requestBody="{\"engineType\":\"$engineType\",\"version\":\"$version\",\"creator\":\"$creator\",\"configKey\":\"$configKey\"}" + curl -i -X DELETE $requestUrl -H "Accept: application/json" -H "Content-Type: application/json" -H "Token-Code:BML-AUTH" -H "Token-User:$user" -d "$requestBody" +} + +add() +{ + requestUrl="$gatewayUrl/api/rest_j/v1/configuration/keyvalue" + requestBody="{\"engineType\":\"$engineType\",\"version\":\"$version\",\"creator\":\"$creator\",\"configKey\":\"$configKey\",\"configValue\":\"$configValue\",\"force\":\"$force\",\"user\":\"$user\"}" + curl -i -X POST $requestUrl -H "Accept: application/json" -H "Content-Type: application/json" -H "Token-Code:BML-AUTH" -H "Token-User:hadoop" -d "$requestBody" +} + +case $COMMAND in + add|get|delete) + $COMMAND + ;; + *) + print_usage + exit 2 + ;; +esac diff --git a/linkis-dist/package/admin/linkis_task_res_log_clear.sh b/linkis-dist/package/admin/linkis_task_res_log_clear.sh new file mode 100644 index 0000000000..4272633e23 --- /dev/null +++ b/linkis-dist/package/admin/linkis_task_res_log_clear.sh @@ -0,0 +1,54 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +#!/bin/bash +expiredDays=365 +resultSetRootDir=/tmp/linkis +logRootDir=/tmp/linkis +userResultSetDir=$(hdfs dfs -ls $resultSetRootDir | awk '{print $8}') +realLogRootDir=$logRootDir/log +echo userResultSetDirs: $userResultSetDir +echo realLogRootDir: $realLogRootDir + +if [ -z $LINKIS_LOG_DIR ];then + expiredFileRecordDir=${LINKIS_HOME}/expiredFileRecord +else + expiredFileRecordDir=$LINKIS_LOG_DIR/expiredFileRecord +fi + +function createExpiredFileRecoredDir(){ + if [ ! -d $expiredFileRecordDir ];then + mkdir -p $expiredFileRecordDir + fi +} + +createExpiredFileRecoredDir +expireDate=$(date -d -${expiredDays}day +%Y-%m-%d) +expireResultSetFile=$expiredFileRecordDir/linkis_expire_resultset_dir_${expireDate}.txt +expireLogFile=$expiredFileRecordDir/linkis_expire_log_dir_${expireDate}.txt + +hdfs dfs -ls $realLogRootDir | awk '$8 ~ /.*linkis\/log\/[0-9|\-|\_]*/ {cmd = "date -d -12month +%Y-%m-%d";cmd | getline oneMonthAgo;if($6 < oneMonthAgo) print $8}' >> $expireLogFile + +for i in $userResultSetDir +do + hdfs dfs -ls $i/linkis | awk '$8 ~ /.*linkis\/[0-9\-]{10}/ {cmd = "date -d -12month +%Y-%m-%d";cmd | getline oneMonthAgo;if($6 < oneMonthAgo) print $8}' >> $expireResultSetFile +done + +cat $expireLogFile | xargs -n 1000 hdfs dfs -rm -r -f + +cat $expireResultSetFile | xargs -n 1000 hdfs dfs -rm -r -f + + diff --git a/linkis-dist/package/sbin/ext/linkis-et-monitor b/linkis-dist/package/sbin/ext/linkis-et-monitor new file mode 100644 index 0000000000..e0c78c487d --- /dev/null +++ b/linkis-dist/package/sbin/ext/linkis-et-monitor @@ -0,0 +1,43 @@ +#!/usr/bin/env bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# description: monitor start cmd +# + + +source $LINKIS_CONF_DIR/linkis-env.sh +export SERVER_SUFFIX="linkis-extensions/linkis-et-monitor" + +export SERVER_CLASS=org.apache.linkis.monitor.LinksMonitorApplication + +if test -z "$MONITOR_HEAP_SIZE" + then + if test -z "$SERVER_HEAP_SIZE" + then + export SERVER_HEAP_SIZE="512M" + fi +else + export SERVER_HEAP_SIZE=$MONITOR_HEAP_SIZE +fi + +#export DEBUG_PORT= + +export COMMON_START_BIN=$LINKIS_HOME/sbin/ext/linkis-common-start +if [[ ! -f "${COMMON_START_BIN}" ]]; then + echo "The $COMMON_START_BIN does not exist!" + exit 1 +else + sh $COMMON_START_BIN +fi \ No newline at end of file diff --git a/linkis-dist/src/main/assembly/distribution.xml b/linkis-dist/src/main/assembly/distribution.xml index 57fd07d4d6..a506c99b18 100644 --- a/linkis-dist/src/main/assembly/distribution.xml +++ b/linkis-dist/src/main/assembly/distribution.xml @@ -261,6 +261,21 @@ + + + + + + ../linkis-extensions/linkis-et-monitor/target/out/lib + + + linkis-package/lib/linkis-extensions/linkis-et-monitor + + + **/* + + + diff --git a/linkis-extensions/linkis-et-monitor/src/main/assembly/distribution.xml b/linkis-extensions/linkis-et-monitor/src/main/assembly/distribution.xml index 647b84c833..e606ed79f2 100644 --- a/linkis-extensions/linkis-et-monitor/src/main/assembly/distribution.xml +++ b/linkis-extensions/linkis-et-monitor/src/main/assembly/distribution.xml @@ -23,6 +23,7 @@ dir false + linkis-et-monitor diff --git a/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/LinksMonitorResourceApplication.java b/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/LinksMonitorApplication.java similarity index 96% rename from linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/LinksMonitorResourceApplication.java rename to linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/LinksMonitorApplication.java index 941c722f77..8f503dcf65 100644 --- a/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/LinksMonitorResourceApplication.java +++ b/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/LinksMonitorApplication.java @@ -24,7 +24,7 @@ @EnableScheduling @SpringBootApplication -public class LinksMonitorResourceApplication { +public class LinksMonitorApplication { public static void main(String[] args) throws ReflectiveOperationException { LinkisBaseServerApp.main(args); diff --git a/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/bml/cleaner/dao/VersionDao.java b/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/bml/cleaner/dao/VersionDao.java index 5f498500eb..b4492c95ac 100644 --- a/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/bml/cleaner/dao/VersionDao.java +++ b/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/bml/cleaner/dao/VersionDao.java @@ -17,7 +17,6 @@ package org.apache.linkis.monitor.bml.cleaner.dao; -import org.apache.linkis.monitor.bml.cleaner.vo.CleanResourceVo; import org.apache.linkis.monitor.bml.cleaner.entity.CleanedResourceVersion; import org.apache.linkis.monitor.bml.cleaner.entity.ResourceVersion; import org.apache.linkis.monitor.bml.cleaner.vo.CleanResourceVo; diff --git a/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/bml/cleaner/service/VersionService.java b/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/bml/cleaner/service/VersionService.java index 5cc2ee217e..cc1a45c9e6 100644 --- a/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/bml/cleaner/service/VersionService.java +++ b/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/bml/cleaner/service/VersionService.java @@ -17,9 +17,8 @@ package org.apache.linkis.monitor.bml.cleaner.service; -import org.apache.linkis.monitor.bml.cleaner.entity.CleanedResourceVersion; -import org.apache.linkis.monitor.bml.cleaner.entity.CleanedResourceVersion; import org.apache.linkis.common.io.FsPath; +import org.apache.linkis.monitor.bml.cleaner.entity.CleanedResourceVersion; import org.apache.linkis.storage.fs.FileSystem; import java.io.IOException; diff --git a/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/bml/cleaner/service/impl/CleanerServiceImpl.java b/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/bml/cleaner/service/impl/CleanerServiceImpl.java index 0005cc08fe..ec2ed2d758 100644 --- a/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/bml/cleaner/service/impl/CleanerServiceImpl.java +++ b/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/bml/cleaner/service/impl/CleanerServiceImpl.java @@ -17,13 +17,13 @@ package org.apache.linkis.monitor.bml.cleaner.service.impl; +import org.apache.linkis.common.io.FsPath; import org.apache.linkis.monitor.bml.cleaner.dao.VersionDao; import org.apache.linkis.monitor.bml.cleaner.entity.CleanedResourceVersion; import org.apache.linkis.monitor.bml.cleaner.entity.ResourceVersion; import org.apache.linkis.monitor.bml.cleaner.service.CleanerService; -import org.apache.linkis.monitor.bml.cleaner.vo.CleanResourceVo; import org.apache.linkis.monitor.bml.cleaner.service.VersionService; -import org.apache.linkis.common.io.FsPath; +import org.apache.linkis.monitor.bml.cleaner.vo.CleanResourceVo; import org.apache.linkis.monitor.constants.Constants; import org.apache.linkis.storage.FSFactory; import org.apache.linkis.storage.fs.FileSystem; @@ -74,10 +74,8 @@ public void setVersionDao(VersionDao versionDao) { public void clean() { previous = new Date( - System.currentTimeMillis() - - (Long) Constants.BML_PREVIOUS_INTERVAL_TIME_DAYS().getValue() - * 86400 - * 1000); + System.currentTimeMillis() + - (Long) Constants.BML_PREVIOUS_INTERVAL_TIME_DAYS().getValue() * 86400 * 1000); if ((Integer) Constants.BML_VERSION_MAX_NUM().getValue() - (Integer) Constants.BML_VERSION_KEEP_NUM().getValue() @@ -90,8 +88,7 @@ public void clean() { logger.info("need cleaned resource count:{}", needCleanResources.size()); fs = (FileSystem) - FSFactory.getFs( - StorageUtils.HDFS, StorageConfiguration.HDFS_ROOT_USER.getValue()); + FSFactory.getFs(StorageUtils.HDFS, StorageConfiguration.HDFS_ROOT_USER.getValue()); for (CleanResourceVo resourceVo : needCleanResources) { String minVersion = versionDao.getMinKeepVersion( diff --git a/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/bml/cleaner/service/impl/VersionServiceImpl.java b/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/bml/cleaner/service/impl/VersionServiceImpl.java index 8e78baa31b..be9e5b70ea 100644 --- a/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/bml/cleaner/service/impl/VersionServiceImpl.java +++ b/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/bml/cleaner/service/impl/VersionServiceImpl.java @@ -17,12 +17,10 @@ package org.apache.linkis.monitor.bml.cleaner.service.impl; -import org.apache.linkis.monitor.bml.cleaner.dao.VersionDao; -import org.apache.linkis.monitor.bml.cleaner.entity.CleanedResourceVersion; +import org.apache.linkis.common.io.FsPath; import org.apache.linkis.monitor.bml.cleaner.dao.VersionDao; import org.apache.linkis.monitor.bml.cleaner.entity.CleanedResourceVersion; import org.apache.linkis.monitor.bml.cleaner.service.VersionService; -import org.apache.linkis.common.io.FsPath; import org.apache.linkis.storage.fs.FileSystem; import org.springframework.beans.factory.annotation.Autowired; @@ -34,8 +32,7 @@ @Service public class VersionServiceImpl implements VersionService { - @Autowired - VersionDao versionDao; + @Autowired VersionDao versionDao; public void setVersionDao(VersionDao versionDao) { this.versionDao = versionDao; diff --git a/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/instance/dao/InsLabelRelationDao.java b/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/instance/dao/InsLabelRelationDao.java index 9e573f42d4..d805c1a99b 100644 --- a/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/instance/dao/InsLabelRelationDao.java +++ b/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/instance/dao/InsLabelRelationDao.java @@ -17,7 +17,6 @@ package org.apache.linkis.monitor.instance.dao; -import org.apache.linkis.monitor.instance.entity.InsPersistenceLabel; import org.apache.linkis.monitor.instance.entity.InsPersistenceLabel; import java.util.List; diff --git a/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/instance/dao/InstanceInfoDao.java b/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/instance/dao/InstanceInfoDao.java index 136df3a252..973801a37a 100644 --- a/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/instance/dao/InstanceInfoDao.java +++ b/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/instance/dao/InstanceInfoDao.java @@ -19,7 +19,6 @@ import org.apache.linkis.common.ServiceInstance; import org.apache.linkis.monitor.instance.entity.InstanceInfo; -import org.apache.linkis.monitor.instance.entity.InstanceInfo; import org.apache.ibatis.annotations.Param; diff --git a/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/instance/dao/InstanceLabelDao.java b/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/instance/dao/InstanceLabelDao.java index 4476444460..361bebf8ba 100644 --- a/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/instance/dao/InstanceLabelDao.java +++ b/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/instance/dao/InstanceLabelDao.java @@ -17,7 +17,6 @@ package org.apache.linkis.monitor.instance.dao; -import org.apache.linkis.monitor.instance.entity.InsPersistenceLabel; import org.apache.linkis.monitor.instance.entity.InsPersistenceLabel; import java.util.List; diff --git a/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/jobhistory/dao/JobHistoryMapper.java b/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/jobhistory/dao/JobHistoryMapper.java index a3cf6fa823..ebd213cfb3 100644 --- a/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/jobhistory/dao/JobHistoryMapper.java +++ b/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/jobhistory/dao/JobHistoryMapper.java @@ -17,7 +17,6 @@ package org.apache.linkis.monitor.jobhistory.dao; -import org.apache.linkis.monitor.jobhistory.entity.JobHistory; import org.apache.linkis.monitor.jobhistory.entity.JobHistory; import org.apache.ibatis.annotations.Param; diff --git a/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/jobhistory/entity/JobHistory.java b/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/jobhistory/entity/JobHistory.java index ae378ec202..e2499d328d 100644 --- a/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/jobhistory/entity/JobHistory.java +++ b/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/jobhistory/entity/JobHistory.java @@ -17,7 +17,6 @@ package org.apache.linkis.monitor.jobhistory.entity; -import org.apache.linkis.monitor.jobhistory.QueryUtils; import org.apache.linkis.monitor.jobhistory.QueryUtils; import java.util.Date; diff --git a/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scheduled/EntranceTaskMonitor.java b/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scheduled/EntranceTaskMonitor.java index d28f6a73f8..7da7f0e4f2 100644 --- a/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scheduled/EntranceTaskMonitor.java +++ b/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scheduled/EntranceTaskMonitor.java @@ -20,9 +20,9 @@ import org.apache.linkis.common.ServiceInstance; import org.apache.linkis.governance.common.conf.GovernanceCommonConf; import org.apache.linkis.monitor.config.MonitorConfig; +import org.apache.linkis.monitor.constants.Constants; import org.apache.linkis.monitor.entity.IndexEntity; import org.apache.linkis.monitor.until.HttpsUntils; -import org.apache.linkis.monitor.constants.Constants; import org.apache.linkis.monitor.utils.alert.AlertDesc; import org.apache.linkis.monitor.utils.alert.ims.MonitorAlertUtils; import org.apache.linkis.monitor.utils.alert.ims.PooledImsAlertUtils; diff --git a/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scheduled/JobHistoryMonitor.java b/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scheduled/JobHistoryMonitor.java index 93dd028c3c..c4bd65a90b 100644 --- a/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scheduled/JobHistoryMonitor.java +++ b/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scheduled/JobHistoryMonitor.java @@ -17,10 +17,13 @@ package org.apache.linkis.monitor.scheduled; -import org.apache.linkis.monitor.jobhistory.JobHistoryDataFetcher; import org.apache.linkis.monitor.config.MonitorConfig; -import org.apache.linkis.monitor.until.CacheUtils; -import org.apache.linkis.monitor.scan.app.factory.MapperFactory; +import org.apache.linkis.monitor.constants.Constants; +import org.apache.linkis.monitor.core.pac.DataFetcher; +import org.apache.linkis.monitor.core.scanner.AnomalyScanner; +import org.apache.linkis.monitor.core.scanner.DefaultScanner; +import org.apache.linkis.monitor.factory.MapperFactory; +import org.apache.linkis.monitor.jobhistory.JobHistoryDataFetcher; import org.apache.linkis.monitor.jobhistory.errorcode.JobHistoryErrCodeRule; import org.apache.linkis.monitor.jobhistory.errorcode.JobHistoryErrorCodeAlertSender; import org.apache.linkis.monitor.jobhistory.jobtime.JobTimeExceedAlertSender; @@ -31,10 +34,7 @@ import org.apache.linkis.monitor.jobhistory.runtime.CommonRunTimeAlertSender; import org.apache.linkis.monitor.jobhistory.runtime.JobHistoryRunTimeAlertSender; import org.apache.linkis.monitor.jobhistory.runtime.JobHistoryRunTimeRule; -import org.apache.linkis.monitor.constants.Constants; -import org.apache.linkis.monitor.core.pac.DataFetcher; -import org.apache.linkis.monitor.core.scanner.AnomalyScanner; -import org.apache.linkis.monitor.core.scanner.DefaultScanner; +import org.apache.linkis.monitor.until.CacheUtils; import org.apache.linkis.monitor.utils.alert.AlertDesc; import org.apache.linkis.monitor.utils.alert.ims.ImsAlertDesc; import org.apache.linkis.monitor.utils.alert.ims.MonitorAlertUtils; @@ -50,19 +50,19 @@ import org.slf4j.Logger; /** - * * jobHistory monitor - * 1.errorCode: Monitor the error code - * 2.userLabel: tenant label monitoring,scan the execution data within the first 20 minutes, and judge the labels field of the data + * * jobHistory monitor 1.errorCode: Monitor the error code 2.userLabel: tenant label + * monitoring,scan the execution data within the first 20 minutes, and judge the labels field of the + * data * - * 3.jobResultRunTime: Scan the execution data within the first 20 minutes, and judge the completed - * tasks. If the parm field in the jobhistory contains (task.notification.conditions) and the result - * of executing the task is (Succeed, Failed, Canceled, Timeout, ALL) any one of them, an alarm is - * triggered and the result of the job is that it has ended. All three are indispensable + *

3.jobResultRunTime: Scan the execution data within the first 20 minutes, and judge the + * completed tasks. If the parm field in the jobhistory contains (task.notification.conditions) and + * the result of executing the task is (Succeed, Failed, Canceled, Timeout, ALL) any one of them, an + * alarm is triggered and the result of the job is that it has ended. All three are indispensable * - * 4.jobResultRunTimeForDSS: Scan the execution data within the first 20 minutes, scan the tasks + *

4.jobResultRunTimeForDSS: Scan the execution data within the first 20 minutes, scan the tasks * that have been marked for notification, if the task has ended, a notification will be initiated * - * 5.jobHistoryUnfinishedScan: monitor the status of the execution task, scan the data outside 12 + *

5.jobHistoryUnfinishedScan: monitor the status of the execution task, scan the data outside 12 * hours and within 24 hours */ @Component @@ -231,10 +231,7 @@ private static List generateFetchers( new String[] {String.valueOf(ps), String.valueOf(pe), String.valueOf(id), timeType}; ret.add(new JobHistoryDataFetcher(fetcherArgs, MapperFactory.getJobHistoryMapper())); logger.info( - "Generated dataFetcher for startTime: " - + new Date(ps) - + ". EndTime: " - + new Date(pe)); + "Generated dataFetcher for startTime: " + new Date(ps) + ". EndTime: " + new Date(pe)); pe = pe - maxIntervalMs; } return ret; diff --git a/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scheduled/ResourceMonitor.java b/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scheduled/ResourceMonitor.java index 0bc8fe7024..ac3046501c 100644 --- a/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scheduled/ResourceMonitor.java +++ b/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scheduled/ResourceMonitor.java @@ -19,9 +19,9 @@ import org.apache.linkis.common.utils.ByteTimeUtils; import org.apache.linkis.monitor.config.MonitorConfig; +import org.apache.linkis.monitor.constants.Constants; import org.apache.linkis.monitor.entity.IndexEntity; import org.apache.linkis.monitor.until.HttpsUntils; -import org.apache.linkis.monitor.constants.Constants; import org.apache.linkis.monitor.utils.alert.AlertDesc; import org.apache.linkis.monitor.utils.alert.ims.MonitorAlertUtils; import org.apache.linkis.monitor.utils.alert.ims.PooledImsAlertUtils; diff --git a/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scheduled/UserModeMonitor.java b/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scheduled/UserModeMonitor.java index e0a7f9f0af..163ada39f1 100644 --- a/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scheduled/UserModeMonitor.java +++ b/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scheduled/UserModeMonitor.java @@ -22,8 +22,8 @@ import org.apache.linkis.httpclient.dws.config.DWSClientConfig; import org.apache.linkis.manager.label.constant.LabelKeyConstant; import org.apache.linkis.monitor.config.MonitorConfig; -import org.apache.linkis.monitor.until.HttpsUntils; import org.apache.linkis.monitor.constants.Constants; +import org.apache.linkis.monitor.until.HttpsUntils; import org.apache.linkis.monitor.utils.alert.AlertDesc; import org.apache.linkis.monitor.utils.alert.ims.MonitorAlertUtils; import org.apache.linkis.monitor.utils.alert.ims.PooledImsAlertUtils; diff --git a/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/until/HttpsUntils.java b/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/until/HttpsUntils.java index fd3f584b34..5bbd53cdc5 100644 --- a/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/until/HttpsUntils.java +++ b/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/until/HttpsUntils.java @@ -22,11 +22,11 @@ import org.apache.linkis.httpclient.dws.authentication.TokenAuthenticationStrategy; import org.apache.linkis.httpclient.dws.config.DWSClientConfig; import org.apache.linkis.httpclient.dws.config.DWSClientConfigBuilder; -import org.apache.linkis.monitor.entity.IndexEntity; -import org.apache.linkis.monitor.config.MonitorConfig; import org.apache.linkis.monitor.client.MonitorHTTPClient; import org.apache.linkis.monitor.client.MonitorHTTPClientClientImpl; +import org.apache.linkis.monitor.config.MonitorConfig; import org.apache.linkis.monitor.constants.Constants; +import org.apache.linkis.monitor.entity.IndexEntity; import org.apache.linkis.monitor.request.EmsListAction; import org.apache.linkis.monitor.request.EntranceTaskAction; import org.apache.linkis.monitor.response.EntranceTaskResult; diff --git a/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/LinkisJobHistoryScanSpringConfiguration.scala b/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/LinkisJobHistoryScanSpringConfiguration.scala index c35d14163b..4154661f07 100644 --- a/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/LinkisJobHistoryScanSpringConfiguration.scala +++ b/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/LinkisJobHistoryScanSpringConfiguration.scala @@ -20,6 +20,7 @@ package org.apache.linkis.monitor import org.apache.linkis.monitor.factory.MapperFactory import org.apache.linkis.monitor.instance.dao.InstanceInfoDao import org.apache.linkis.monitor.jobhistory.dao.JobHistoryMapper + import org.springframework.beans.factory.annotation.Autowired import org.springframework.context.annotation.{ComponentScan, Configuration} diff --git a/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/client/MonitorHTTPClient.scala b/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/client/MonitorHTTPClient.scala index c8f125b738..4caccd73a3 100644 --- a/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/client/MonitorHTTPClient.scala +++ b/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/client/MonitorHTTPClient.scala @@ -21,11 +21,7 @@ import org.apache.linkis.httpclient.authentication.AuthenticationStrategy import org.apache.linkis.httpclient.dws.authentication.StaticAuthenticationStrategy import org.apache.linkis.httpclient.dws.config.{DWSClientConfig, DWSClientConfigBuilder} import org.apache.linkis.httpclient.response.Result -import org.apache.linkis.monitor.request.{ - EmsListAction, - EntranceTaskAction, - MonitorResourceAction -} +import org.apache.linkis.monitor.request.{EmsListAction, EntranceTaskAction, MonitorResourceAction} import org.apache.linkis.monitor.response.EntranceTaskResult import org.apache.linkis.ujes.client.response.EmsListResult diff --git a/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/constants/Constants.scala b/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/constants/Constants.scala index 3f2aa6c78b..07d66690d6 100644 --- a/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/constants/Constants.scala +++ b/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/constants/Constants.scala @@ -99,7 +99,6 @@ object Constants { val THREAD_TIME_OUT_IM = "thread.monitor.timeout.im." val JOB_RESULT_IM = "jobhistory.result.monitor.im." - val BML_VERSION_MAX_NUM: CommonVars[Int] = CommonVars[Int]("linkis.monitor.bml.cleaner.version.max.num", 50) @@ -112,10 +111,7 @@ object Constants { val BML_CLEAN_ONCE_RESOURCE_LIMIT_NUM: CommonVars[Int] = CommonVars[Int]("linkis.monitor.bml.cleaner.once.limit.num", 100) - val BML_TRASH_PATH_PREFIX: CommonVars[String] = CommonVars[String]( - "linkis.monitor.bml.trash.prefix.path", - "hdfs:///tmp/linkis/trash/bml_trash" - ) - + val BML_TRASH_PATH_PREFIX: CommonVars[String] = + CommonVars[String]("linkis.monitor.bml.trash.prefix.path", "hdfs:///tmp/linkis/trash/bml_trash") } diff --git a/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/core/scanner/AbstractScanner.scala b/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/core/scanner/AbstractScanner.scala index e478ca9781..4f207cd697 100644 --- a/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/core/scanner/AbstractScanner.scala +++ b/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/core/scanner/AbstractScanner.scala @@ -18,9 +18,9 @@ package org.apache.linkis.monitor.core.scanner import org.apache.linkis.common.utils.Logging -import org.apache.linkis.monitor.jobhistory.exception.AnomalyScannerException import org.apache.linkis.monitor.core.ob.{Event, Observer} import org.apache.linkis.monitor.core.pac._ +import org.apache.linkis.monitor.jobhistory.exception.AnomalyScannerException import java.util import java.util.concurrent.CopyOnWriteArrayList @@ -159,8 +159,6 @@ abstract class AbstractScanner extends AnomalyScanner with Logging { * 1. should be non-blocking 2. keeps calling scanOneIteration() and analyzeOneIteration() utils * stop() is called */ - override def start(): Unit = { - - } + override def start(): Unit = {} } diff --git a/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/factory/MapperFactory.scala b/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/factory/MapperFactory.scala index 82bc9b56b4..eb503c52aa 100644 --- a/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/factory/MapperFactory.scala +++ b/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/factory/MapperFactory.scala @@ -17,9 +17,12 @@ package org.apache.linkis.monitor.factory -import org.apache.linkis.monitor.instance.dao.{InsLabelRelationDao, InstanceInfoDao, InstanceLabelDao} +import org.apache.linkis.monitor.instance.dao.{ + InsLabelRelationDao, + InstanceInfoDao, + InstanceLabelDao +} import org.apache.linkis.monitor.jobhistory.dao.JobHistoryMapper -import org.apache.linkis.monitor.scan.app.instance.dao.{InsLabelRelationDao, InstanceInfoDao, InstanceLabelDao} object MapperFactory { diff --git a/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/jobhistory/JobHistoryDataFetcher.scala b/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/jobhistory/JobHistoryDataFetcher.scala index 08315ded75..fb371a658d 100644 --- a/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/jobhistory/JobHistoryDataFetcher.scala +++ b/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/jobhistory/JobHistoryDataFetcher.scala @@ -20,10 +20,11 @@ package org.apache.linkis.monitor.jobhistory import org.apache.linkis.common.utils.{Logging, Utils} import org.apache.linkis.monitor.constants.Constants import org.apache.linkis.monitor.core.pac.AbstractDataFetcher -import org.apache.commons.lang3.StringUtils import org.apache.linkis.monitor.jobhistory.dao.JobHistoryMapper import org.apache.linkis.monitor.jobhistory.exception.AnomalyScannerException +import org.apache.commons.lang3.StringUtils + import java.util import java.util.Date diff --git a/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/jobhistory/errorcode/JobHistoryErrCodeRule.scala b/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/jobhistory/errorcode/JobHistoryErrCodeRule.scala index 73f457d909..d354f7f7b0 100644 --- a/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/jobhistory/errorcode/JobHistoryErrCodeRule.scala +++ b/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/jobhistory/errorcode/JobHistoryErrCodeRule.scala @@ -18,12 +18,13 @@ package org.apache.linkis.monitor.jobhistory.errorcode import org.apache.linkis.common.utils.Logging -import org.apache.linkis.monitor.jobhistory.entity.JobHistory -import org.apache.linkis.monitor.until.CacheUtils import org.apache.linkis.monitor.core.ob.Observer import org.apache.linkis.monitor.core.pac.{AbstractScanRule, ScannedData} +import org.apache.linkis.monitor.jobhistory.entity.JobHistory +import org.apache.linkis.monitor.until.CacheUtils import java.util + import scala.collection.JavaConverters._ /** diff --git a/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/jobhistory/errorcode/JobHistoryErrorCodeAlertSender.scala b/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/jobhistory/errorcode/JobHistoryErrorCodeAlertSender.scala index 4f7865ba34..7f3d8e10cb 100644 --- a/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/jobhistory/errorcode/JobHistoryErrorCodeAlertSender.scala +++ b/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/jobhistory/errorcode/JobHistoryErrorCodeAlertSender.scala @@ -18,13 +18,14 @@ package org.apache.linkis.monitor.jobhistory.errorcode import org.apache.linkis.common.utils.Logging +import org.apache.linkis.monitor.core.ob.{Event, Observer} import org.apache.linkis.monitor.jobhistory.entity.JobHistory import org.apache.linkis.monitor.jobhistory.exception.AnomalyScannerException -import org.apache.linkis.monitor.core.ob.{Event, Observer} import org.apache.linkis.monitor.utils.alert.AlertDesc import org.apache.linkis.monitor.utils.alert.ims.{ImsAlertDesc, PooledImsAlertUtils} import java.util + import scala.collection.JavaConverters._ class JobHistoryErrorCodeAlertSender(alerts: util.Map[String, AlertDesc]) diff --git a/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/jobhistory/jobtime/JobTimeExceedAlertSender.scala b/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/jobhistory/jobtime/JobTimeExceedAlertSender.scala index 4b1fafe155..0a53142eb7 100644 --- a/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/jobhistory/jobtime/JobTimeExceedAlertSender.scala +++ b/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/jobhistory/jobtime/JobTimeExceedAlertSender.scala @@ -18,15 +18,16 @@ package org.apache.linkis.monitor.jobhistory.jobtime import org.apache.linkis.common.utils.{Logging, Utils} -import org.apache.linkis.monitor.jobhistory.entity.JobHistory -import org.apache.linkis.monitor.jobhistory.exception.AnomalyScannerException import org.apache.linkis.monitor.config.MonitorConfig import org.apache.linkis.monitor.core.ob.{Event, Observer} +import org.apache.linkis.monitor.jobhistory.entity.JobHistory +import org.apache.linkis.monitor.jobhistory.exception.AnomalyScannerException import org.apache.linkis.monitor.utils.alert.AlertDesc import org.apache.linkis.monitor.utils.alert.ims.{ImsAlertDesc, PooledImsAlertUtils} import java.text.MessageFormat import java.util + import scala.collection.JavaConverters._ import scala.collection.mutable.ArrayBuffer diff --git a/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/jobhistory/jobtime/JobTimeExceedRule.scala b/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/jobhistory/jobtime/JobTimeExceedRule.scala index 289b4fde61..f788173e43 100644 --- a/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/jobhistory/jobtime/JobTimeExceedRule.scala +++ b/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/jobhistory/jobtime/JobTimeExceedRule.scala @@ -18,15 +18,16 @@ package org.apache.linkis.monitor.jobhistory.jobtime import org.apache.linkis.common.utils.Logging -import org.apache.linkis.monitor.jobhistory.entity.JobHistory -import org.apache.linkis.monitor.jobhistory.exception.AnomalyScannerException -import org.apache.linkis.monitor.until.CacheUtils import org.apache.linkis.monitor.constants.Constants import org.apache.linkis.monitor.core.ob.Observer import org.apache.linkis.monitor.core.pac.{AbstractScanRule, ScannedData} +import org.apache.linkis.monitor.jobhistory.entity.JobHistory +import org.apache.linkis.monitor.jobhistory.exception.AnomalyScannerException +import org.apache.linkis.monitor.until.CacheUtils import java.util import java.util.Locale + import scala.collection.JavaConverters._ /** diff --git a/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/jobhistory/labels/JobHistoryLabelsAlertSender.scala b/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/jobhistory/labels/JobHistoryLabelsAlertSender.scala index fb0ab0258d..7876156b1d 100644 --- a/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/jobhistory/labels/JobHistoryLabelsAlertSender.scala +++ b/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/jobhistory/labels/JobHistoryLabelsAlertSender.scala @@ -18,15 +18,16 @@ package org.apache.linkis.monitor.jobhistory.labels import org.apache.linkis.common.utils.Logging -import org.apache.linkis.monitor.jobhistory.entity.JobHistory -import org.apache.linkis.monitor.jobhistory.exception.AnomalyScannerException import org.apache.linkis.monitor.constants.Constants import org.apache.linkis.monitor.core.ob.{Event, Observer} +import org.apache.linkis.monitor.jobhistory.entity.JobHistory +import org.apache.linkis.monitor.jobhistory.exception.AnomalyScannerException import org.apache.linkis.monitor.utils.alert.AlertDesc import org.apache.linkis.monitor.utils.alert.ims.{PooledImsAlertUtils, UserLabelAlertUtils} import org.apache.linkis.server.BDPJettyServerHelper import java.util + import scala.collection.JavaConverters._ import scala.collection.mutable.ArrayBuffer diff --git a/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/jobhistory/labels/JobHistoryLabelsRule.scala b/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/jobhistory/labels/JobHistoryLabelsRule.scala index c50fd127e4..70d309b883 100644 --- a/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/jobhistory/labels/JobHistoryLabelsRule.scala +++ b/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/jobhistory/labels/JobHistoryLabelsRule.scala @@ -21,14 +21,17 @@ import org.apache.linkis.common.utils.Logging import org.apache.linkis.monitor.constants.Constants import org.apache.linkis.monitor.core.ob.Observer import org.apache.linkis.monitor.core.pac.{AbstractScanRule, ScannedData} +import org.apache.linkis.monitor.jobhistory.entity.JobHistory +import org.apache.linkis.monitor.until.CacheUtils import org.apache.linkis.server.BDPJettyServerHelper + import org.apache.commons.lang3.StringUtils import java.util + import scala.collection.JavaConverters._ + import com.google.common.collect.HashBiMap -import org.apache.linkis.monitor.jobhistory.entity.JobHistory -import org.apache.linkis.monitor.until.CacheUtils /** * Scan the execution data within the previous 20 minutes and judge the labels field of the data. diff --git a/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/jobhistory/runtime/CommonJobRunTimeRule.scala b/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/jobhistory/runtime/CommonJobRunTimeRule.scala index 46644861d0..77d904fe4d 100644 --- a/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/jobhistory/runtime/CommonJobRunTimeRule.scala +++ b/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/jobhistory/runtime/CommonJobRunTimeRule.scala @@ -21,10 +21,12 @@ import org.apache.linkis.common.utils.Logging import org.apache.linkis.monitor.constants.Constants import org.apache.linkis.monitor.core.ob.Observer import org.apache.linkis.monitor.core.pac.{AbstractScanRule, ScannedData} -import org.apache.commons.lang3.StringUtils import org.apache.linkis.monitor.jobhistory.entity.JobHistory +import org.apache.commons.lang3.StringUtils + import java.util + import scala.collection.JavaConverters._ /** @@ -55,9 +57,7 @@ class CommonJobRunTimeRule(hitObserver: Observer) d match { case jobHistory: JobHistory => if ( - Constants.FINISHED_JOB_STATUS.contains( - jobHistory.getStatus.toUpperCase() - ) + Constants.FINISHED_JOB_STATUS.contains(jobHistory.getStatus.toUpperCase()) && StringUtils.isNotBlank(jobHistory.getObserveInfo) ) { alertData.add(jobHistory) diff --git a/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/jobhistory/runtime/CommonRunTimeAlertSender.scala b/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/jobhistory/runtime/CommonRunTimeAlertSender.scala index f54a96e0e2..2380891463 100644 --- a/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/jobhistory/runtime/CommonRunTimeAlertSender.scala +++ b/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/jobhistory/runtime/CommonRunTimeAlertSender.scala @@ -20,16 +20,18 @@ package org.apache.linkis.monitor.jobhistory.runtime import org.apache.linkis.common.utils.Logging import org.apache.linkis.monitor.constants.Constants import org.apache.linkis.monitor.core.ob.{Event, Observer} +import org.apache.linkis.monitor.jobhistory.entity.JobHistory +import org.apache.linkis.monitor.jobhistory.exception.AnomalyScannerException import org.apache.linkis.monitor.utils.alert.ims.{MonitorAlertUtils, PooledImsAlertUtils} import org.apache.linkis.server.BDPJettyServerHelper + import org.apache.commons.collections.MapUtils -import org.apache.linkis.monitor.jobhistory.entity.JobHistory -import org.apache.linkis.monitor.jobhistory.exception.AnomalyScannerException import java.net.InetAddress import java.text.SimpleDateFormat import java.util import java.util.Date + import scala.collection.JavaConverters._ class CommonRunTimeAlertSender() extends Observer with Logging { diff --git a/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/jobhistory/runtime/JobHistoryRunTimeAlertSender.scala b/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/jobhistory/runtime/JobHistoryRunTimeAlertSender.scala index 7469d3dcca..a1e870c7c5 100644 --- a/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/jobhistory/runtime/JobHistoryRunTimeAlertSender.scala +++ b/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/jobhistory/runtime/JobHistoryRunTimeAlertSender.scala @@ -18,13 +18,14 @@ package org.apache.linkis.monitor.jobhistory.runtime import org.apache.linkis.common.utils.Logging -import org.apache.linkis.monitor.jobhistory.entity.JobHistory -import org.apache.linkis.monitor.jobhistory.exception.AnomalyScannerException import org.apache.linkis.monitor.constants.Constants import org.apache.linkis.monitor.core.ob.{Event, Observer} +import org.apache.linkis.monitor.jobhistory.entity.JobHistory +import org.apache.linkis.monitor.jobhistory.exception.AnomalyScannerException import org.apache.linkis.monitor.utils.alert.ims.{MonitorAlertUtils, PooledImsAlertUtils} import java.util + import scala.collection.JavaConverters._ /** diff --git a/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/jobhistory/runtime/JobHistoryRunTimeRule.scala b/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/jobhistory/runtime/JobHistoryRunTimeRule.scala index be3b5f0362..d350bc3ace 100644 --- a/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/jobhistory/runtime/JobHistoryRunTimeRule.scala +++ b/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/jobhistory/runtime/JobHistoryRunTimeRule.scala @@ -18,15 +18,16 @@ package org.apache.linkis.monitor.jobhistory.runtime import org.apache.linkis.common.utils.Logging -import org.apache.linkis.monitor.jobhistory.entity.JobHistory -import org.apache.linkis.monitor.until.CacheUtils import org.apache.linkis.monitor.constants.Constants import org.apache.linkis.monitor.core.ob.Observer import org.apache.linkis.monitor.core.pac.{AbstractScanRule, ScannedData} +import org.apache.linkis.monitor.jobhistory.entity.JobHistory +import org.apache.linkis.monitor.until.CacheUtils import org.apache.linkis.protocol.utils.TaskUtils import org.apache.linkis.server.BDPJettyServerHelper import java.util + import scala.collection.JavaConverters._ class JobHistoryRunTimeRule(hitObserver: Observer) @@ -51,11 +52,7 @@ class JobHistoryRunTimeRule(hitObserver: Observer) for (d <- sd.getData().asScala) { d match { case jobHistory: JobHistory => - if ( - Constants.FINISHED_JOB_STATUS.contains( - jobHistory.getStatus.toUpperCase() - ) - ) { + if (Constants.FINISHED_JOB_STATUS.contains(jobHistory.getStatus.toUpperCase())) { val parmsMap: util.Map[String, scala.AnyRef] = BDPJettyServerHelper.gson.fromJson( jobHistory.getParams, classOf[util.Map[String, scala.AnyRef]] diff --git a/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/utils/alert/ims/ImsAlertDesc.scala b/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/utils/alert/ims/ImsAlertDesc.scala index 30cbc0b211..06ef57f629 100644 --- a/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/utils/alert/ims/ImsAlertDesc.scala +++ b/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/utils/alert/ims/ImsAlertDesc.scala @@ -18,16 +18,19 @@ package org.apache.linkis.monitor.utils.alert.ims import org.apache.linkis.monitor.constants.Constants +import org.apache.linkis.monitor.utils.ScanUtils import org.apache.linkis.monitor.utils.alert.AlertDesc + import org.apache.commons.collections.CollectionUtils import org.apache.commons.lang3.StringUtils import java.util import java.util.HashSet + import scala.collection.JavaConverters._ + import ImsAlertLevel.ImsAlertLevel import ImsAlertWay.ImsAlertWay -import org.apache.linkis.monitor.utils.ScanUtils case class ImsAlertDesc( var subSystemId: String, diff --git a/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/utils/alert/ims/MonitorAlertUtils.scala b/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/utils/alert/ims/MonitorAlertUtils.scala index f230b9cdf7..67c1b0358f 100644 --- a/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/utils/alert/ims/MonitorAlertUtils.scala +++ b/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/utils/alert/ims/MonitorAlertUtils.scala @@ -19,7 +19,9 @@ package org.apache.linkis.monitor.utils.alert.ims import org.apache.linkis.common.utils.{JsonUtils, Logging, Utils} import org.apache.linkis.monitor.constants.Constants +import org.apache.linkis.monitor.jobhistory.exception.AnomalyScannerException import org.apache.linkis.monitor.utils.alert.AlertDesc + import org.apache.commons.io.IOUtils import org.apache.commons.lang3.StringUtils import org.apache.commons.lang3.exception.ExceptionUtils @@ -28,10 +30,11 @@ import java.io.{BufferedReader, File, FileInputStream, InputStream, InputStreamR import java.text.SimpleDateFormat import java.util import java.util.Properties + import scala.collection.JavaConverters._ + import com.fasterxml.jackson.databind.{DeserializationFeature, ObjectMapper} import com.fasterxml.jackson.module.scala.DefaultScalaModule -import org.apache.linkis.monitor.jobhistory.exception.AnomalyScannerException object MonitorAlertUtils extends Logging { @@ -77,10 +80,7 @@ object MonitorAlertUtils extends Logging { properties.asScala } - def getAlerts( - prefix: String, - params: util.Map[String, String] - ): util.Map[String, AlertDesc] = { + def getAlerts(prefix: String, params: util.Map[String, String]): util.Map[String, AlertDesc] = { val ret = new util.HashMap[String, AlertDesc]() for ((k: String, v: String) <- properties) { diff --git a/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/utils/alert/ims/UserLabelAlertUtils.scala b/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/utils/alert/ims/UserLabelAlertUtils.scala index 250103412f..64a587a21c 100644 --- a/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/utils/alert/ims/UserLabelAlertUtils.scala +++ b/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/utils/alert/ims/UserLabelAlertUtils.scala @@ -19,7 +19,9 @@ package org.apache.linkis.monitor.utils.alert.ims import org.apache.linkis.common.utils.{JsonUtils, Logging, Utils} import org.apache.linkis.monitor.constants.Constants +import org.apache.linkis.monitor.jobhistory.exception.AnomalyScannerException import org.apache.linkis.monitor.utils.alert.AlertDesc + import org.apache.commons.io.IOUtils import org.apache.commons.lang3.StringUtils import org.apache.commons.lang3.exception.ExceptionUtils @@ -28,10 +30,11 @@ import java.io.{BufferedReader, File, FileInputStream, InputStream, InputStreamR import java.text.SimpleDateFormat import java.util import java.util.Properties + import scala.collection.JavaConverters._ + import com.fasterxml.jackson.databind.{DeserializationFeature, ObjectMapper} import com.fasterxml.jackson.module.scala.DefaultScalaModule -import org.apache.linkis.monitor.jobhistory.exception.AnomalyScannerException object UserLabelAlertUtils extends Logging {