From 42435c50c9282a1b5e47ecd934f0abadc08fc734 Mon Sep 17 00:00:00 2001 From: peacewong Date: Mon, 4 Sep 2023 19:19:07 +0800 Subject: [PATCH 1/6] bak --- linkis-extensions/linkis-et-monitor/pom.xml | 128 ++++++++ .../src/main/assembly/distribution.xml | 297 ++++++++++++++++++ .../app/LinkisDirtyDataCleanApplication.java | 111 +++++++ .../app/LinkisJobHistoryScanApplication.java | 172 ++++++++++ .../app/LinksMonitorResourceApplication.java | 32 ++ .../app/instance/dao/InsLabelRelationDao.java | 35 +++ .../app/instance/dao/InstanceInfoDao.java | 36 +++ .../app/instance/dao/InstanceLabelDao.java | 41 +++ .../instance/entity/InsPersistenceLabel.java | 82 +++++ .../entity/InsPersistenceLabelValue.java | 59 ++++ .../app/instance/entity/InstanceInfo.java | 92 ++++++ .../scan/app/jobhistory/QueryUtils.java | 31 ++ .../app/jobhistory/dao/JobHistoryMapper.java | 68 ++++ .../app/jobhistory/entity/JobHistory.java | 221 +++++++++++++ .../exception/AnomalyScannerException.java | 37 +++ .../exception/DirtyDataCleanException.java | 37 +++ .../app/monitor/config/ListenerConfig.java | 46 +++ .../app/monitor/config/MonitorConfig.java | 71 +++++ .../app/monitor/config/ScheduledConfig.java | 39 +++ .../app/monitor/entity/ChatbotEntity.java | 87 +++++ .../scan/app/monitor/entity/EngineEntity.java | 53 ++++ .../app/monitor/entity/EntranceEntity.java | 63 ++++ .../scan/app/monitor/entity/IndexEntity.java | 85 +++++ .../scan/app/monitor/scheduled/BmlClear.java | 47 +++ .../app/monitor/scheduled/EcRecordClear.java | 53 ++++ .../monitor/scheduled/JobHistoryClear.java | 53 ++++ .../monitor/scheduled/JobHistoryMonitor.java | 264 ++++++++++++++++ .../monitor/scheduled/ResourceMonitor.java | 151 +++++++++ .../app/monitor/scheduled/TaskLogClear.java | 53 ++++ .../app/monitor/scheduled/TaskMonitor.java | 183 +++++++++++ .../monitor/scheduled/UserModeMonitor.java | 156 +++++++++ .../app/monitor/scheduled/ValidatorClear.java | 51 +++ .../scan/app/monitor/until/CacheUtils.java | 35 +++ .../scan/app/monitor/until/HttpsUntils.java | 183 +++++++++++ .../scan/app/monitor/until/ThreadUtils.java | 68 ++++ .../mapper/common/InsLabelRelationMapper.xml | 67 ++++ .../mapper/common/InstanceInfoMapper.xml | 53 ++++ .../mapper/common/InstanceLabelMapper.xml | 57 ++++ .../mapper/common/JobHistoryMapper.xml | 172 ++++++++++ ...kisJobHistoryScanSpringConfiguration.scala | 49 +++ .../scan/app/factory/MapperFactory.scala | 64 ++++ .../jobhistory/JobHistoryDataFetcher.scala | 111 +++++++ .../errorcode/JobHistoryErrCodeHitEvent.scala | 23 ++ .../errorcode/JobHistoryErrCodeRule.scala | 80 +++++ .../JobHistoryErrorCodeAlertSender.scala | 97 ++++++ .../jobtime/JobTimeExceedAlertSender.scala | 112 +++++++ .../jobtime/JobTimeExceedHitEvent.scala | 23 ++ .../jobtime/JobTimeExceedRule.scala | 103 ++++++ .../labels/JobHistoryLabelsAlertSender.scala | 69 ++++ .../labels/JobHistoryLabelsHitEvent.scala | 22 ++ .../labels/JobHistoryLabelsRule.scala | 112 +++++++ .../runtime/CommonJobRunTimeRule.scala | 79 +++++ .../runtime/CommonRunTimeAlertSender.scala | 88 ++++++ .../runtime/CommonRunTimeHitEvent.scala | 22 ++ .../JobHistoryRunTimeAlertSender.scala | 73 +++++ .../runtime/JobHistoryRunTimeHitEvent.scala | 22 ++ .../runtime/JobHistoryRunTimeRule.scala | 82 +++++ .../scan/client/MonitorHTTPClient.scala | 117 +++++++ .../client/MonitorHTTPClientClientImpl.scala | 39 +++ .../scan/client/MonitorResourceClient.scala | 112 +++++++ .../client/MonitorResourceClientImpl.scala | 39 +++ .../monitor/scan/constants/Constants.scala | 113 +++++++ .../scan/constants/ScanOperatorEnum.scala | 24 ++ .../linkis/monitor/scan/core/ob/Event.scala | 29 ++ .../monitor/scan/core/ob/Observer.scala | 27 ++ .../scan/core/ob/SingleObserverEvent.java | 43 +++ .../scan/core/pac/AbstractDataFetcher.scala | 30 ++ .../scan/core/pac/AbstractScanRule.scala | 52 +++ .../scan/core/pac/BaseScannedData.scala | 27 ++ .../monitor/scan/core/pac/DataFetcher.scala | 40 +++ .../monitor/scan/core/pac/ScanBuffer.scala | 43 +++ .../monitor/scan/core/pac/ScanRule.scala | 50 +++ .../monitor/scan/core/pac/ScannedData.scala | 27 ++ .../scan/core/scanner/AbstractScanner.scala | 168 ++++++++++ .../scan/core/scanner/AnomalyScanner.scala | 95 ++++++ .../scan/core/scanner/DefaultScanner.scala | 29 ++ .../monitor/scan/request/EmsListAction.scala | 74 +++++ .../scan/request/EntranceTaskAction.scala | 76 +++++ .../scan/request/MonitorResourceAction.scala | 22 ++ .../monitor/scan/request/UserAction.scala | 26 ++ .../scan/response/EntranceTaskResult.scala | 36 +++ .../scan/response/MonitorResourceResult.scala | 33 ++ .../linkis/monitor/scan/utils/ScanUtils.java | 71 +++++ .../monitor/scan/utils/alert/AlertDesc.scala | 26 ++ .../scan/utils/alert/AlertSender.scala | 30 ++ .../scan/utils/alert/PooledAlertSender.scala | 112 +++++++ .../scan/utils/alert/ims/ImsAlertDesc.scala | 155 +++++++++ .../scan/utils/alert/ims/ImsAlertLevel.scala | 29 ++ .../scan/utils/alert/ims/ImsAlertWay.scala | 27 ++ .../scan/utils/alert/ims/ImsRequest.scala | 36 +++ ...storyScanImsAlertPropFileParserUtils.scala | 132 ++++++++ .../utils/alert/ims/MonitorAlertUtils.scala | 229 ++++++++++++++ .../alert/ims/PooledImsAlertSender.scala | 110 +++++++ .../utils/alert/ims/PooledImsAlertUtils.scala | 115 +++++++ .../utils/alert/ims/UserLabelAlertUtils.scala | 126 ++++++++ .../monitor/scan/utils/log/LogUtils.scala | 25 ++ ...ryScanImsAlertPropFileParserUtilsTest.java | 48 +++ .../LinkisJobHistoryScanApplicationTest.java | 38 +++ .../utils/alert/PooledImsAlertSenderTest.java | 67 ++++ .../utils/alert/PooledImsAlertUtilsTest.java | 49 +++ .../linkis-et-jobhistory-scan-ims.properties | 6 + .../linkis-et-jobhistory-scan.properties | 14 + .../src/test/resources/log4j2-console.xml | 46 +++ linkis-extensions/pom.xml | 1 + 104 files changed, 7663 insertions(+) create mode 100644 linkis-extensions/linkis-et-monitor/pom.xml create mode 100644 linkis-extensions/linkis-et-monitor/src/main/assembly/distribution.xml create mode 100644 linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/LinkisDirtyDataCleanApplication.java create mode 100644 linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/LinkisJobHistoryScanApplication.java create mode 100644 linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/LinksMonitorResourceApplication.java create mode 100644 linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/instance/dao/InsLabelRelationDao.java create mode 100644 linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/instance/dao/InstanceInfoDao.java create mode 100644 linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/instance/dao/InstanceLabelDao.java create mode 100644 linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/instance/entity/InsPersistenceLabel.java create mode 100644 linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/instance/entity/InsPersistenceLabelValue.java create mode 100644 linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/instance/entity/InstanceInfo.java create mode 100644 linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/jobhistory/QueryUtils.java create mode 100644 linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/jobhistory/dao/JobHistoryMapper.java create mode 100644 linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/jobhistory/entity/JobHistory.java create mode 100644 linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/jobhistory/exception/AnomalyScannerException.java create mode 100644 linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/jobhistory/exception/DirtyDataCleanException.java create mode 100644 linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/monitor/config/ListenerConfig.java create mode 100644 linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/monitor/config/MonitorConfig.java create mode 100644 linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/monitor/config/ScheduledConfig.java create mode 100644 linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/monitor/entity/ChatbotEntity.java create mode 100644 linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/monitor/entity/EngineEntity.java create mode 100644 linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/monitor/entity/EntranceEntity.java create mode 100644 linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/monitor/entity/IndexEntity.java create mode 100644 linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/monitor/scheduled/BmlClear.java create mode 100644 linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/monitor/scheduled/EcRecordClear.java create mode 100644 linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/monitor/scheduled/JobHistoryClear.java create mode 100644 linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/monitor/scheduled/JobHistoryMonitor.java create mode 100644 linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/monitor/scheduled/ResourceMonitor.java create mode 100644 linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/monitor/scheduled/TaskLogClear.java create mode 100644 linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/monitor/scheduled/TaskMonitor.java create mode 100644 linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/monitor/scheduled/UserModeMonitor.java create mode 100644 linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/monitor/scheduled/ValidatorClear.java create mode 100644 linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/monitor/until/CacheUtils.java create mode 100644 linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/monitor/until/HttpsUntils.java create mode 100644 linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/monitor/until/ThreadUtils.java create mode 100644 linkis-extensions/linkis-et-monitor/src/main/resources/mapper/common/InsLabelRelationMapper.xml create mode 100644 linkis-extensions/linkis-et-monitor/src/main/resources/mapper/common/InstanceInfoMapper.xml create mode 100644 linkis-extensions/linkis-et-monitor/src/main/resources/mapper/common/InstanceLabelMapper.xml create mode 100644 linkis-extensions/linkis-et-monitor/src/main/resources/mapper/common/JobHistoryMapper.xml create mode 100644 linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/app/LinkisJobHistoryScanSpringConfiguration.scala create mode 100644 linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/app/factory/MapperFactory.scala create mode 100644 linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/app/jobhistory/JobHistoryDataFetcher.scala create mode 100644 linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/app/jobhistory/errorcode/JobHistoryErrCodeHitEvent.scala create mode 100644 linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/app/jobhistory/errorcode/JobHistoryErrCodeRule.scala create mode 100644 linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/app/jobhistory/errorcode/JobHistoryErrorCodeAlertSender.scala create mode 100644 linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/app/jobhistory/jobtime/JobTimeExceedAlertSender.scala create mode 100644 linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/app/jobhistory/jobtime/JobTimeExceedHitEvent.scala create mode 100644 linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/app/jobhistory/jobtime/JobTimeExceedRule.scala create mode 100644 linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/app/jobhistory/labels/JobHistoryLabelsAlertSender.scala create mode 100644 linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/app/jobhistory/labels/JobHistoryLabelsHitEvent.scala create mode 100644 linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/app/jobhistory/labels/JobHistoryLabelsRule.scala create mode 100644 linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/app/jobhistory/runtime/CommonJobRunTimeRule.scala create mode 100644 linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/app/jobhistory/runtime/CommonRunTimeAlertSender.scala create mode 100644 linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/app/jobhistory/runtime/CommonRunTimeHitEvent.scala create mode 100644 linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/app/jobhistory/runtime/JobHistoryRunTimeAlertSender.scala create mode 100644 linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/app/jobhistory/runtime/JobHistoryRunTimeHitEvent.scala create mode 100644 linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/app/jobhistory/runtime/JobHistoryRunTimeRule.scala create mode 100644 linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/client/MonitorHTTPClient.scala create mode 100644 linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/client/MonitorHTTPClientClientImpl.scala create mode 100644 linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/client/MonitorResourceClient.scala create mode 100644 linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/client/MonitorResourceClientImpl.scala create mode 100644 linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/constants/Constants.scala create mode 100644 linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/constants/ScanOperatorEnum.scala create mode 100644 linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/core/ob/Event.scala create mode 100644 linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/core/ob/Observer.scala create mode 100644 linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/core/ob/SingleObserverEvent.java create mode 100644 linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/core/pac/AbstractDataFetcher.scala create mode 100644 linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/core/pac/AbstractScanRule.scala create mode 100644 linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/core/pac/BaseScannedData.scala create mode 100644 linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/core/pac/DataFetcher.scala create mode 100644 linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/core/pac/ScanBuffer.scala create mode 100644 linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/core/pac/ScanRule.scala create mode 100644 linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/core/pac/ScannedData.scala create mode 100644 linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/core/scanner/AbstractScanner.scala create mode 100644 linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/core/scanner/AnomalyScanner.scala create mode 100644 linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/core/scanner/DefaultScanner.scala create mode 100644 linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/request/EmsListAction.scala create mode 100644 linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/request/EntranceTaskAction.scala create mode 100644 linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/request/MonitorResourceAction.scala create mode 100644 linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/request/UserAction.scala create mode 100644 linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/response/EntranceTaskResult.scala create mode 100644 linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/response/MonitorResourceResult.scala create mode 100644 linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/utils/ScanUtils.java create mode 100644 linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/utils/alert/AlertDesc.scala create mode 100644 linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/utils/alert/AlertSender.scala create mode 100644 linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/utils/alert/PooledAlertSender.scala create mode 100644 linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/utils/alert/ims/ImsAlertDesc.scala create mode 100644 linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/utils/alert/ims/ImsAlertLevel.scala create mode 100644 linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/utils/alert/ims/ImsAlertWay.scala create mode 100644 linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/utils/alert/ims/ImsRequest.scala create mode 100644 linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/utils/alert/ims/JobHistoryScanImsAlertPropFileParserUtils.scala create mode 100644 linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/utils/alert/ims/MonitorAlertUtils.scala create mode 100644 linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/utils/alert/ims/PooledImsAlertSender.scala create mode 100644 linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/utils/alert/ims/PooledImsAlertUtils.scala create mode 100644 linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/utils/alert/ims/UserLabelAlertUtils.scala create mode 100644 linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/utils/log/LogUtils.scala create mode 100644 linkis-extensions/linkis-et-monitor/src/test/java/org/apache/linkis/monitor/scan/app/JobHistoryScanImsAlertPropFileParserUtilsTest.java create mode 100644 linkis-extensions/linkis-et-monitor/src/test/java/org/apache/linkis/monitor/scan/app/LinkisJobHistoryScanApplicationTest.java create mode 100644 linkis-extensions/linkis-et-monitor/src/test/java/org/apache/linkis/monitor/scan/utils/alert/PooledImsAlertSenderTest.java create mode 100644 linkis-extensions/linkis-et-monitor/src/test/java/org/apache/linkis/monitor/scan/utils/alert/PooledImsAlertUtilsTest.java create mode 100644 linkis-extensions/linkis-et-monitor/src/test/resources/linkis-et-jobhistory-scan-ims.properties create mode 100644 linkis-extensions/linkis-et-monitor/src/test/resources/linkis-et-jobhistory-scan.properties create mode 100644 linkis-extensions/linkis-et-monitor/src/test/resources/log4j2-console.xml diff --git a/linkis-extensions/linkis-et-monitor/pom.xml b/linkis-extensions/linkis-et-monitor/pom.xml new file mode 100644 index 0000000000..5e62fa60b5 --- /dev/null +++ b/linkis-extensions/linkis-et-monitor/pom.xml @@ -0,0 +1,128 @@ + + + + 4.0.0 + + org.apache.linkis + linkis + ${revision} + ../../pom.xml + + + linkis-et-monitor + + + + org.apache.linkis + linkis-httpclient + ${project.version} + + + org.apache.linkis + linkis-mybatis + ${project.version} + + + + org.apache.linkis + linkis-wedatasphere-common + ${project.version} + + + + org.apache.linkis + linkis-module + ${project.version} + provided + + + + org.apache.linkis + linkis-gateway-httpclient-support + ${project.version} + + + + org.apache.linkis + linkis-common + ${project.version} + + + + org.springframework + spring-test + test + + + + junit + junit + 4.12 + test + + + org.apache.linkis + linkis-computation-client + ${project.version} + + + org.apache.linkis + linkis-bml-cleaner + ${project.version} + + + org.apache.linkis + linkis-storage + ${project.version} + provided + + + org.apache.linkis + linkis-rpc + ${project.version} + + + + + + net.alchim31.maven + scala-maven-plugin + + + org.apache.maven.plugins + maven-assembly-plugin + false + + false + out + false + false + + src/main/assembly/distribution.xml + + + + + make-assembly + + single + + package + + + + + + + diff --git a/linkis-extensions/linkis-et-monitor/src/main/assembly/distribution.xml b/linkis-extensions/linkis-et-monitor/src/main/assembly/distribution.xml new file mode 100644 index 0000000000..8cc9e7ea7b --- /dev/null +++ b/linkis-extensions/linkis-et-monitor/src/main/assembly/distribution.xml @@ -0,0 +1,297 @@ + + + + + linkis-et-monitor + + dir + + false + + + + + + lib + true + true + false + false + true + + + antlr:antlr:jar + aopalliance:aopalliance:jar + asm:asm:jar + cglib:cglib:jar + com.amazonaws:aws-java-sdk-autoscaling:jar + com.amazonaws:aws-java-sdk-core:jar + com.amazonaws:aws-java-sdk-ec2:jar + com.amazonaws:aws-java-sdk-route53:jar + com.amazonaws:aws-java-sdk-sts:jar + com.amazonaws:jmespath-java:jar + com.fasterxml.jackson.core:jackson-annotations:jar + com.fasterxml.jackson.core:jackson-core:jar + com.fasterxml.jackson.core:jackson-databind:jar + com.fasterxml.jackson.dataformat:jackson-dataformat-cbor:jar + com.fasterxml.jackson.datatype:jackson-datatype-jdk8:jar + com.fasterxml.jackson.datatype:jackson-datatype-jsr310:jar + com.fasterxml.jackson.jaxrs:jackson-jaxrs-base:jar + com.fasterxml.jackson.jaxrs:jackson-jaxrs-json-provider:jar + com.fasterxml.jackson.module:jackson-module-jaxb-annotations:jar + com.fasterxml.jackson.module:jackson-module-parameter-names:jar + com.fasterxml.jackson.module:jackson-module-paranamer:jar + com.fasterxml.jackson.module:jackson-module-scala_2.11:jar + com.github.andrewoma.dexx:dexx-collections:jar + com.github.vlsi.compactmap:compactmap:jar + com.google.code.findbugs:annotations:jar + com.google.code.findbugs:jsr305:jar + com.google.code.gson:gson:jar + com.google.guava:guava:jar + com.google.inject:guice:jar + com.google.protobuf:protobuf-java:jar + com.netflix.archaius:archaius-core:jar + com.netflix.eureka:eureka-client:jar + com.netflix.eureka:eureka-core:jar + com.netflix.hystrix:hystrix-core:jar + com.netflix.netflix-commons:netflix-commons-util:jar + com.netflix.netflix-commons:netflix-eventbus:jar + com.netflix.netflix-commons:netflix-infix:jar + com.netflix.netflix-commons:netflix-statistics:jar + com.netflix.ribbon:ribbon:jar + com.netflix.ribbon:ribbon-core:jar + com.netflix.ribbon:ribbon-eureka:jar + com.netflix.ribbon:ribbon-httpclient:jar + com.netflix.ribbon:ribbon-loadbalancer:jar + com.netflix.ribbon:ribbon-transport:jar + com.netflix.servo:servo-core:jar + com.ning:async-http-client:jar + com.sun.jersey.contribs:jersey-apache-client4:jar + com.sun.jersey:jersey-client:jar + com.sun.jersey:jersey-core:jar + com.sun.jersey:jersey-json:jar + com.sun.jersey:jersey-server:jar + com.sun.jersey:jersey-servlet:jar + com.sun.xml.bind:jaxb-impl:jar + com.thoughtworks.paranamer:paranamer:jar + com.thoughtworks.xstream:xstream:jar + org.apache.linkis:linkis-common:jar + org.apache.linkis:linkis-module:jar + commons-beanutils:commons-beanutils:jar + commons-beanutils:commons-beanutils-core:jar + commons-cli:commons-cli:jar + commons-codec:commons-codec:jar + commons-collections:commons-collections:jar + commons-configuration:commons-configuration:jar + commons-daemon:commons-daemon:jar + commons-dbcp:commons-dbcp:jar + commons-digester:commons-digester:jar + commons-httpclient:commons-httpclient:jar + commons-io:commons-io:jar + commons-jxpath:commons-jxpath:jar + commons-lang:commons-lang:jar + commons-logging:commons-logging:jar + commons-net:commons-net:jar + commons-pool:commons-pool:jar + io.micrometer:micrometer-core:jar + io.netty:netty:jar + io.netty:netty-all:jar + io.netty:netty-buffer:jar + io.netty:netty-codec:jar + io.netty:netty-codec-http:jar + io.netty:netty-common:jar + io.netty:netty-handler:jar + io.netty:netty-transport:jar + io.netty:netty-transport-native-epoll:jar + io.reactivex:rxjava:jar + io.reactivex:rxnetty:jar + io.reactivex:rxnetty-contexts:jar + io.reactivex:rxnetty-servo:jar + javax.activation:activation:jar + javax.annotation:javax.annotation-api:jar + javax.inject:javax.inject:jar + javax.servlet:javax.servlet-api:jar + javax.servlet.jsp:jsp-api:jar + javax.validation:validation-api:jar + javax.websocket:javax.websocket-api:jar + javax.ws.rs:javax.ws.rs-api:jar + javax.xml.bind:jaxb-api:jar + javax.xml.stream:stax-api:jar + joda-time:joda-time:jar + log4j:log4j:jar + mysql:mysql-connector-java:jar + net.databinder.dispatch:dispatch-core_2.11:jar + net.databinder.dispatch:dispatch-json4s-jackson_2.11:jar + org.antlr:antlr-runtime:jar + org.antlr:stringtemplate:jar + org.apache.commons:commons-compress:jar + org.apache.commons:commons-math:jar + org.apache.commons:commons-math3:jar + org.apache.curator:curator-client:jar + org.apache.curator:curator-framework:jar + org.apache.curator:curator-recipes:jar + org.apache.directory.api:api-asn1-api:jar + org.apache.directory.api:api-util:jar + org.apache.directory.server:apacheds-i18n:jar + org.apache.directory.server:apacheds-kerberos-codec:jar + org.apache.hadoop:hadoop-annotations:jar + org.apache.hadoop:hadoop-auth:jar + org.apache.hadoop:hadoop-common:jar + org.apache.hadoop:hadoop-hdfs:jar + org.apache.htrace:htrace-core:jar + org.apache.httpcomponents:httpclient:jar + org.apache.httpcomponents:httpcore:jar + org.apache.logging.log4j:log4j-api:jar + org.apache.logging.log4j:log4j-core:jar + org.apache.logging.log4j:log4j-jul:jar + org.apache.logging.log4j:log4j-slf4j-impl:jar + org.apache.zookeeper:zookeeper:jar + org.aspectj:aspectjweaver:jar + org.bouncycastle:bcpkix-jdk15on:jar + org.bouncycastle:bcprov-jdk15on:jar + org.codehaus.jackson:jackson-jaxrs:jar + org.codehaus.jackson:jackson-xc:jar + org.codehaus.jettison:jettison:jar + org.codehaus.woodstox:stax2-api:jar + org.codehaus.woodstox:woodstox-core-asl:jar + org.eclipse.jetty:jetty-annotations:jar + org.eclipse.jetty:jetty-client:jar + org.eclipse.jetty:jetty-continuation:jar + org.eclipse.jetty:jetty-http:jar + org.eclipse.jetty:jetty-io:jar + org.eclipse.jetty:jetty-jndi:jar + org.eclipse.jetty:jetty-plus:jar + org.eclipse.jetty:jetty-security:jar + org.eclipse.jetty:jetty-server:jar + org.eclipse.jetty:jetty-servlet:jar + org.eclipse.jetty:jetty-servlets:jar + org.eclipse.jetty:jetty-util:jar + org.eclipse.jetty:jetty-webapp:jar + org.eclipse.jetty:jetty-xml:jar + org.eclipse.jetty.websocket:javax-websocket-client-impl:jar + org.eclipse.jetty.websocket:javax-websocket-server-impl:jar + org.eclipse.jetty.websocket:websocket-api:jar + org.eclipse.jetty.websocket:websocket-client:jar + org.eclipse.jetty.websocket:websocket-common:jar + org.eclipse.jetty.websocket:websocket-server:jar + org.eclipse.jetty.websocket:websocket-servlet:jar + org.fusesource.leveldbjni:leveldbjni-all:jar + org.glassfish.hk2:class-model:jar + org.glassfish.hk2:config-types:jar + org.glassfish.hk2.external:aopalliance-repackaged:jar + org.glassfish.hk2.external:asm-all-repackaged:jar + org.glassfish.hk2.external:bean-validator:jar + org.glassfish.hk2.external:javax.inject:jar + org.glassfish.hk2:hk2:jar + org.glassfish.hk2:hk2-api:jar + org.glassfish.hk2:hk2-config:jar + org.glassfish.hk2:hk2-core:jar + org.glassfish.hk2:hk2-locator:jar + org.glassfish.hk2:hk2-runlevel:jar + org.glassfish.hk2:hk2-utils:jar + org.glassfish.hk2:osgi-resource-locator:jar + org.glassfish.hk2:spring-bridge:jar + org.glassfish.jersey.bundles:jaxrs-ri:jar + org.glassfish.jersey.bundles.repackaged:jersey-guava:jar + org.glassfish.jersey.containers:jersey-container-servlet:jar + org.glassfish.jersey.containers:jersey-container-servlet-core:jar + org.glassfish.jersey.core:jersey-client:jar + org.glassfish.jersey.core:jersey-common:jar + org.glassfish.jersey.core:jersey-server:jar + org.glassfish.jersey.ext:jersey-entity-filtering:jar + org.glassfish.jersey.ext:jersey-spring3:jar + org.glassfish.jersey.media:jersey-media-jaxb:jar + org.glassfish.jersey.media:jersey-media-json-jackson:jar + org.glassfish.jersey.media:jersey-media-multipart:jar + org.hdrhistogram:HdrHistogram:jar + org.javassist:javassist:jar + org.json4s:json4s-ast_2.11:jar + org.json4s:json4s-core_2.11:jar + org.json4s:json4s-jackson_2.11:jar + org.jsoup:jsoup:jar + org.jvnet.mimepull:mimepull:jar + org.jvnet:tiger-types:jar + org.latencyutils:LatencyUtils:jar + org.mortbay.jasper:apache-el:jar + org.mortbay.jetty:jetty:jar + org.mortbay.jetty:jetty-util:jar + org.ow2.asm:asm-analysis:jar + org.ow2.asm:asm-commons:jar + org.ow2.asm:asm-tree:jar + org.reflections:reflections:jar + org.scala-lang.modules:scala-parser-combinators_2.11:jar + org.scala-lang.modules:scala-xml_2.11:jar + org.scala-lang:scala-compiler:jar + org.scala-lang:scala-library:jar + org.scala-lang:scala-reflect:jar + org.scala-lang:scalap:jar + org.slf4j:jul-to-slf4j:jar + org.slf4j:slf4j-api:jar + org.springframework.boot:spring-boot:jar + org.springframework.boot:spring-boot-actuator:jar + org.springframework.boot:spring-boot-actuator-autoconfigure:jar + org.springframework.boot:spring-boot-autoconfigure:jar + org.springframework.boot:spring-boot-starter:jar + org.springframework.boot:spring-boot-starter-actuator:jar + org.springframework.boot:spring-boot-starter-aop:jar + org.springframework.boot:spring-boot-starter-jetty:jar + org.springframework.boot:spring-boot-starter-json:jar + org.springframework.boot:spring-boot-starter-log4j2:jar + org.springframework.boot:spring-boot-starter-web:jar + org.springframework.cloud:spring-cloud-commons:jar + org.springframework.cloud:spring-cloud-config-client:jar + org.springframework.cloud:spring-cloud-context:jar + org.springframework.cloud:spring-cloud-netflix-archaius:jar + org.springframework.cloud:spring-cloud-netflix-core:jar + org.springframework.cloud:spring-cloud-netflix-eureka-client:jar + org.springframework.cloud:spring-cloud-netflix-ribbon:jar + org.springframework.cloud:spring-cloud-starter:jar + org.springframework.cloud:spring-cloud-starter-config:jar + org.springframework.cloud:spring-cloud-starter-eureka:jar + org.springframework.cloud:spring-cloud-starter-netflix-archaius:jar + org.springframework.cloud:spring-cloud-starter-netflix-eureka-client:jar + org.springframework.cloud:spring-cloud-starter-netflix-ribbon:jar + org.springframework.security:spring-security-crypto:jar + org.springframework.security:spring-security-rsa:jar + org.springframework:spring-aop:jar + org.springframework:spring-beans:jar + org.springframework:spring-context:jar + org.springframework:spring-core:jar + org.springframework:spring-expression:jar + org.springframework:spring-jcl:jar + org.springframework:spring-web:jar + org.springframework:spring-webmvc:jar + org.tukaani:xz:jar + org.yaml:snakeyaml:jar + software.amazon.ion:ion-java:jar + xerces:xercesImpl:jar + xmlenc:xmlenc:jar + xmlpull:xmlpull:jar + xpp3:xpp3_min:jar + + + + + + + + + diff --git a/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/LinkisDirtyDataCleanApplication.java b/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/LinkisDirtyDataCleanApplication.java new file mode 100644 index 0000000000..3530ef5139 --- /dev/null +++ b/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/LinkisDirtyDataCleanApplication.java @@ -0,0 +1,111 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.monitor.scan.app; + +import org.apache.linkis.monitor.scan.app.dirtydata.entrance.EntranceDirtyDataHandler; +import org.apache.linkis.monitor.scan.utils.log.LogUtils; +import org.apache.linkis.server.utils.LinkisMainHelper; + +import org.apache.commons.lang3.StringUtils; + +import org.springframework.context.annotation.AnnotationConfigApplicationContext; +import org.springframework.context.support.AbstractApplicationContext; + +import org.slf4j.Logger; + +public class LinkisDirtyDataCleanApplication { + private static final Logger logger = LogUtils.stdOutLogger(); + + /** @param args: args[0]: host args[1] port */ + public static void main(String[] args) throws ReflectiveOperationException { + if (args.length == 0 + || StringUtils.equalsIgnoreCase(args[0], "help") + || StringUtils.equalsIgnoreCase(args[0], "--help")) { + LogUtils.stdOutLogger() + .info( + "[help-message]this app cleans entrance dirty-data. args[0]: command-type (entrance/help/...) args[1]: entrance-hostname(not null), args[2]: entrance-port(can be null)"); + return; + } + String serviceName = System.getProperty(LinkisMainHelper.SERVER_NAME_KEY()); + LinkisMainHelper.formatPropertyFiles(serviceName); + + if (StringUtils.equalsIgnoreCase(args[0], "entrance")) { + AbstractApplicationContext context = + new AnnotationConfigApplicationContext(LinkisJobHistoryScanSpringConfiguration.class); + + String host = ""; + String port = ""; + if (args.length > 1) { + host = args[1]; + } + if (args.length > 2) { + port = args[2]; + } + if (args.length > 3) { + printIllegalInput("wrong number of arguments"); + return; + } + try { + removeDirtyEurekaInstance(host, port); + } catch (Exception e) { + LogUtils.stdOutLogger().error("Failed to remove dirty eureka-instance", e); + } + try { + removeDbDirtyData(host, port); + } catch (Exception e) { + LogUtils.stdOutLogger().error("Failed to remove dirty db-data", e); + } + + context.close(); + } else { + LogUtils.stdOutLogger().error("Upsupported command type: " + args[0]); + } + } + + private static void printIllegalInput(String msg) { + LogUtils.stdOutLogger().error("illegal input: " + msg); + LogUtils.stdOutLogger() + .info( + "[help-message] this app cleans entrance dirty-data. args[0]: entrance-hostname, args[1]: entrance-port"); + return; + } + + private static void removeDirtyEurekaInstance(String host, String port) { + if (StringUtils.isBlank(host)) { + printIllegalInput("host cannot be blank"); + return; + } + if (StringUtils.isBlank(port)) { + EntranceDirtyDataHandler.handleEurekaDirtyData(host); + } else { + EntranceDirtyDataHandler.handleEurekaDirtyData(host, port); + } + } + + private static void removeDbDirtyData(String host, String port) { + if (StringUtils.isBlank(host)) { + printIllegalInput("host cannot be blank"); + return; + } + if (StringUtils.isBlank(port)) { + EntranceDirtyDataHandler.handleDbDirtData(host); + } else { + EntranceDirtyDataHandler.handleDbDirtData(host, port); + } + } +} diff --git a/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/LinkisJobHistoryScanApplication.java b/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/LinkisJobHistoryScanApplication.java new file mode 100644 index 0000000000..8b4366805e --- /dev/null +++ b/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/LinkisJobHistoryScanApplication.java @@ -0,0 +1,172 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.monitor.scan.app; + +import org.apache.linkis.monitor.scan.app.factory.MapperFactory; +import org.apache.linkis.monitor.scan.app.jobhistory.JobHistoryDataFetcher; +import org.apache.linkis.monitor.scan.app.jobhistory.errorcode.JobHistoryErrCodeRule; +import org.apache.linkis.monitor.scan.app.jobhistory.errorcode.JobHistoryErrorCodeAlertSender; +import org.apache.linkis.monitor.scan.app.jobhistory.jobtime.JobTimeExceedAlertSender; +import org.apache.linkis.monitor.scan.app.jobhistory.jobtime.JobTimeExceedRule; +import org.apache.linkis.monitor.scan.constants.Constants; +import org.apache.linkis.monitor.scan.core.pac.DataFetcher; +import org.apache.linkis.monitor.scan.core.scanner.AnomalyScanner; +import org.apache.linkis.monitor.scan.core.scanner.DefaultScanner; +import org.apache.linkis.monitor.scan.utils.alert.AlertDesc; +import org.apache.linkis.monitor.scan.utils.alert.ims.ImsAlertDesc; +import org.apache.linkis.monitor.scan.utils.alert.ims.JobHistoryScanImsAlertPropFileParserUtils; +import org.apache.linkis.monitor.scan.utils.log.LogUtils; +import org.apache.linkis.server.utils.LinkisMainHelper; + +import org.apache.commons.lang3.exception.ExceptionUtils; + +import org.springframework.context.annotation.AnnotationConfigApplicationContext; +import org.springframework.context.support.AbstractApplicationContext; + +import java.text.ParseException; +import java.text.SimpleDateFormat; +import java.util.ArrayList; +import java.util.Date; +import java.util.List; +import java.util.Map; + +import org.slf4j.Logger; + +public class LinkisJobHistoryScanApplication { + private static final Logger logger = LogUtils.stdOutLogger(); + + /** @param args: args[0]: startTime, args[1] endTime */ + public static void main(String[] args) throws ReflectiveOperationException { + + String serviceName = System.getProperty(LinkisMainHelper.SERVER_NAME_KEY()); + LinkisMainHelper.formatPropertyFiles(serviceName); + + long intervalMs = Constants.SCAN_INTERVALS_SECONDS() * 1000; + long maxIntervalMs = Constants.MAX_INTERVALS_SECONDS() * 1000; + long endTime = System.currentTimeMillis(); + long startTime = endTime - intervalMs; + /** parse input into timestamp */ + if (args != null && args.length == 2) { + SimpleDateFormat format = new SimpleDateFormat("yyyyMMddHH"); + Long tmpStart; + Long tmpEnd; + try { + tmpStart = format.parse(args[0]).getTime(); + tmpEnd = format.parse(args[1]).getTime(); + } catch (ParseException e) { + logger.info( + "Failed to parse input into startTime and endTime." + ExceptionUtils.getMessage(e)); + return; + } + if (endTime <= startTime) { + logger.info("[error] startTime larger than endTime"); + return; + } + if (tmpStart != null && tmpEnd != null) { + startTime = tmpStart; + endTime = tmpEnd; + } + } + long realIntervals = endTime - startTime < maxIntervalMs ? endTime - startTime : maxIntervalMs; + + runApp(startTime, endTime, realIntervals, maxIntervalMs); + } + + private static void runApp(long startTime, long endTime, long realIntervals, long maxIntervalMs) { + + AbstractApplicationContext context = + new AnnotationConfigApplicationContext(LinkisJobHistoryScanSpringConfiguration.class); + + AnomalyScanner scanner = new DefaultScanner(); + boolean shouldStart = false; + + List fetchers = generateFetchers(startTime, endTime, maxIntervalMs); + if (fetchers == null) { + logger.warn("generated 0 dataFetchers, plz check input"); + return; + } + + Map errorCodeAlerts = + JobHistoryScanImsAlertPropFileParserUtils.getAlerts(Constants.SCAN_PREFIX_ERRORCODE()); + if (errorCodeAlerts == null || errorCodeAlerts.size() == 0) { + logger.info("[INFO] Loaded 0 errorcode alert from alert-rule properties file."); + } else { + logger.info( + "[INFO] Loaded {} errorcode alert from alert-rules properties file.", + errorCodeAlerts.size()); + shouldStart = true; + addIntervalToImsAlerts(errorCodeAlerts, realIntervals); + JobHistoryErrCodeRule jobHistoryErrCodeRule = + new JobHistoryErrCodeRule( + errorCodeAlerts.keySet(), new JobHistoryErrorCodeAlertSender(errorCodeAlerts)); + scanner.addScanRule(jobHistoryErrCodeRule); + } + + Map jobTimeAlerts = + JobHistoryScanImsAlertPropFileParserUtils.getAlerts( + Constants.SCAN_PREFIX_UNFINISHED_JOBTIME_EXCEED_SEC()); + if (jobTimeAlerts == null || jobTimeAlerts.size() == 0) { + logger.info("[INFO] Loaded 0 alerts jobtime alert-rule from alert properties file."); + } else { + logger.info( + "[INFO] Loaded {} alerts jobtime alert-rules from alert properties file.", + jobTimeAlerts.size()); + shouldStart = true; + addIntervalToImsAlerts(jobTimeAlerts, realIntervals); + JobTimeExceedRule jobTimeExceedRule = + new JobTimeExceedRule( + jobTimeAlerts.keySet(), new JobTimeExceedAlertSender(jobTimeAlerts)); + scanner.addScanRule(jobTimeExceedRule); + } + if (shouldStart) { + scanner.addDataFetchers(fetchers); + scanner.run(); + scanner.shutdown(); // wait all alert to be send + } + context.close(); + } + + private static List generateFetchers( + long startTime, long endTime, long maxIntervalMs) { + List ret = new ArrayList<>(); + long pe = endTime; + long ps; + while (pe > startTime) { + ps = pe - maxIntervalMs > startTime ? pe - maxIntervalMs : startTime; + String[] fetcherArgs = new String[] {String.valueOf(ps), String.valueOf(pe)}; + ret.add(new JobHistoryDataFetcher(fetcherArgs, MapperFactory.getJobHistoryMapper())); + logger.info( + "Generated dataFetcher for startTime: " + + new Date(ps).toString() + + ". EndTime: " + + new Date(pe).toString()); + pe = pe - maxIntervalMs; + } + return ret; + } + + private static void addIntervalToImsAlerts(Map alerts, long realIntervals) { + for (AlertDesc alert : alerts.values()) { + if (!(alert instanceof ImsAlertDesc)) { + logger.info("[warn] ignore wrong alert" + alert); + } else { + ((ImsAlertDesc) alert).hitIntervalMs_$eq(realIntervals); + } + } + } +} diff --git a/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/LinksMonitorResourceApplication.java b/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/LinksMonitorResourceApplication.java new file mode 100644 index 0000000000..405d94037b --- /dev/null +++ b/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/LinksMonitorResourceApplication.java @@ -0,0 +1,32 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.monitor.scan.app; + +import org.apache.linkis.LinkisBaseServerApp; + +import org.springframework.boot.autoconfigure.SpringBootApplication; +import org.springframework.scheduling.annotation.EnableScheduling; + +@EnableScheduling +@SpringBootApplication +public class LinksMonitorResourceApplication { + + public static void main(String[] args) throws ReflectiveOperationException { + LinkisBaseServerApp.main(args); + } +} diff --git a/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/instance/dao/InsLabelRelationDao.java b/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/instance/dao/InsLabelRelationDao.java new file mode 100644 index 0000000000..3705288e66 --- /dev/null +++ b/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/instance/dao/InsLabelRelationDao.java @@ -0,0 +1,35 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.monitor.scan.app.instance.dao; + +import org.apache.linkis.monitor.scan.app.instance.entity.InsPersistenceLabel; + +import java.util.List; + +public interface InsLabelRelationDao { + + /** + * Search labels + * + * @param instance instance value (http:port) + * @return + */ + List searchLabelsByInstance(String instance); + + void dropRelationsByInstance(String instance); +} diff --git a/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/instance/dao/InstanceInfoDao.java b/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/instance/dao/InstanceInfoDao.java new file mode 100644 index 0000000000..73a7166a5f --- /dev/null +++ b/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/instance/dao/InstanceInfoDao.java @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.monitor.scan.app.instance.dao; + +import org.apache.linkis.common.ServiceInstance; +import org.apache.linkis.monitor.scan.app.instance.entity.InstanceInfo; + +import org.apache.ibatis.annotations.Param; + +import java.util.List; + +public interface InstanceInfoDao { + + void removeInstance(ServiceInstance instance); + + List getInstanceInfoByApplicationNameAndHostnameFuzzy( + @Param("hostname") String hostname, @Param("applicationName") String applicationName); + + List getInstanceInfoByApplicationNameAndInstanceName( + @Param("instanceName") String instanceName, @Param("applicationName") String applicationName); +} diff --git a/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/instance/dao/InstanceLabelDao.java b/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/instance/dao/InstanceLabelDao.java new file mode 100644 index 0000000000..8f73f0349a --- /dev/null +++ b/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/instance/dao/InstanceLabelDao.java @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.monitor.scan.app.instance.dao; + +import org.apache.linkis.monitor.scan.app.instance.entity.InsPersistenceLabel; + +import java.util.List; + + +public interface InstanceLabelDao { + /** + * Remove label + * + * @param label label entity + */ + void remove(InsPersistenceLabel label); + + /** + * Remove key -> value map from label id + * + * @param labelId + */ + void doRemoveKeyValues(Integer labelId); + + void doRemoveKeyValuesBatch(List labelIds); +} diff --git a/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/instance/entity/InsPersistenceLabel.java b/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/instance/entity/InsPersistenceLabel.java new file mode 100644 index 0000000000..e828328b43 --- /dev/null +++ b/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/instance/entity/InsPersistenceLabel.java @@ -0,0 +1,82 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.monitor.scan.app.instance.entity; + +import java.util.Date; + +public class InsPersistenceLabel { + private Integer id; + private int labelValueSize = -1; + private String stringValue; + private Boolean modifiable = false; + + private String labelKey; + + private String feature; + + private Date updateTime; + private Date createTime; + + public Boolean getModifiable() { + return modifiable; + } + + public void setModifiable(Boolean modifiable) { + this.modifiable = modifiable; + } + + public Integer getId() { + return id; + } + + public void setId(Integer id) { + this.id = id; + } + + public int getLabelValueSize() { + return labelValueSize; + } + + public void setLabelValueSize(int labelValueSize) { + this.labelValueSize = labelValueSize; + } + + public String getStringValue() { + return stringValue; + } + + public void setStringValue(String stringValue) { + this.stringValue = stringValue; + } + + public Date getUpdateTime() { + return updateTime; + } + + public void setUpdateTime(Date updateTime) { + this.updateTime = updateTime; + } + + public Date getCreateTime() { + return createTime; + } + + public void setCreateTime(Date createTime) { + this.createTime = createTime; + } +} diff --git a/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/instance/entity/InsPersistenceLabelValue.java b/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/instance/entity/InsPersistenceLabelValue.java new file mode 100644 index 0000000000..53ab4bd806 --- /dev/null +++ b/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/instance/entity/InsPersistenceLabelValue.java @@ -0,0 +1,59 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.monitor.scan.app.instance.entity; + +public class InsPersistenceLabelValue { + + private Integer labelId; + + private String valueKey; + + private String valueContent; + + public InsPersistenceLabelValue() {} + + public InsPersistenceLabelValue(Integer labelId, String key, String content) { + this.labelId = labelId; + this.valueKey = key; + this.valueContent = content; + } + + public String getValueKey() { + return valueKey; + } + + public void setValueKey(String valueKey) { + this.valueKey = valueKey; + } + + public String getValueContent() { + return valueContent; + } + + public void setValueContent(String valueContent) { + this.valueContent = valueContent; + } + + public Integer getLabelId() { + return labelId; + } + + public void setLabelId(Integer labelId) { + this.labelId = labelId; + } +} diff --git a/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/instance/entity/InstanceInfo.java b/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/instance/entity/InstanceInfo.java new file mode 100644 index 0000000000..a74cfaabb0 --- /dev/null +++ b/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/instance/entity/InstanceInfo.java @@ -0,0 +1,92 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.monitor.scan.app.instance.entity; + +import java.util.ArrayList; +import java.util.Date; +import java.util.List; + +public class InstanceInfo { + /** Automatic increment */ + private Integer id; + + private String applicationName; + + private String instance; + + private Date updateTime; + + private Date createTime; + /** Labels related */ + private List labels = new ArrayList<>(); + + public InstanceInfo() {} + + public InstanceInfo(String applicationName, String instance) { + this.applicationName = applicationName; + this.instance = instance; + } + + public List getLabels() { + return labels; + } + + public void setLabels(List labels) { + this.labels = labels; + } + + public Integer getId() { + return id; + } + + public void setId(Integer id) { + this.id = id; + } + + public String getApplicationName() { + return applicationName; + } + + public void setApplicationName(String applicationName) { + this.applicationName = applicationName; + } + + public String getInstance() { + return instance; + } + + public void setInstance(String instance) { + this.instance = instance; + } + + public Date getUpdateTime() { + return updateTime; + } + + public void setUpdateTime(Date updateTime) { + this.updateTime = updateTime; + } + + public Date getCreateTime() { + return createTime; + } + + public void setCreateTime(Date createTime) { + this.createTime = createTime; + } +} diff --git a/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/jobhistory/QueryUtils.java b/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/jobhistory/QueryUtils.java new file mode 100644 index 0000000000..799eae89a6 --- /dev/null +++ b/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/jobhistory/QueryUtils.java @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.monitor.scan.app.jobhistory; + +import java.text.DateFormat; +import java.text.SimpleDateFormat; +import java.util.Date; + +public class QueryUtils { + + private static DateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss.SSS"); + + public static String dateToString(Date date) { + return dateFormat.format(date); + } +} diff --git a/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/jobhistory/dao/JobHistoryMapper.java b/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/jobhistory/dao/JobHistoryMapper.java new file mode 100644 index 0000000000..b3774860af --- /dev/null +++ b/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/jobhistory/dao/JobHistoryMapper.java @@ -0,0 +1,68 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.monitor.scan.app.jobhistory.dao; + +import org.apache.linkis.monitor.scan.app.jobhistory.entity.JobHistory; + +import org.apache.ibatis.annotations.Param; + +import java.util.Date; +import java.util.List; + + +public interface JobHistoryMapper { + + List selectJobHistory(JobHistory jobReq); + + List search( + @Param("id") Long id, + @Param("umUser") String username, + @Param("status") List status, + @Param("startDate") Date startDate, + @Param("endDate") Date endDate, + @Param("engineType") String engineType); + + void updateIncompleteJobStatusGivenIDList( + @Param("idList") List idList, @Param("targetStatus") String targetStatus); + + void updateJobStatusForInstanceGivenStatusList( + @Param("instanceName") String instanceName, + @Param("statusList") List statusList, + @Param("targetStatus") String targetStatus, + @Param("startDate") Date startDate); + + List searchByCache( + @Param("id") Long id, + @Param("umUser") String username, + @Param("status") List status, + @Param("startDate") Date startDate, + @Param("endDate") Date endDate, + @Param("engineType") String engineType); + + List searchByCacheAndUpdateTime( + @Param("id") Long id, + @Param("umUser") String username, + @Param("status") List status, + @Param("startDate") Date startDate, + @Param("endDate") Date endDate, + @Param("engineType") String engineType); + + Long selectIdByHalfDay(@Param("id") long beginId); + + Long selectMaxId(); +} diff --git a/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/jobhistory/entity/JobHistory.java b/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/jobhistory/entity/JobHistory.java new file mode 100644 index 0000000000..937eff410b --- /dev/null +++ b/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/jobhistory/entity/JobHistory.java @@ -0,0 +1,221 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.monitor.scan.app.jobhistory.entity; + +import org.apache.linkis.monitor.scan.app.jobhistory.QueryUtils; + +import java.util.Date; + +public class JobHistory { + + private Long id; + + private String jobReqId; + + private String submitUser; + + private String executeUser; + + private String source; + + private String labels; + + private String params; + + private String progress; + + private String status; + + private String logPath; + + private Integer errorCode; + + private String errorDesc; + + private Date createdTime; + + private Date updatedTime; + + private String updateTimeMills; + + private String instances; + + private String metrics; + + private String engineType; + + private String executionCode; + + private String observeInfo; + + public Long getId() { + return id; + } + + public void setId(Long id) { + this.id = id; + } + + public String getJobReqId() { + return jobReqId; + } + + public void setJobReqId(String jobReqId) { + this.jobReqId = jobReqId; + } + + public String getSubmitUser() { + return submitUser; + } + + public void setSubmitUser(String submitUser) { + this.submitUser = submitUser; + } + + public String getExecuteUser() { + return executeUser; + } + + public void setExecuteUser(String executeUser) { + this.executeUser = executeUser; + } + + public String getSource() { + return source; + } + + public void setSource(String source) { + this.source = source; + } + + public String getLabels() { + return labels; + } + + public void setLabels(String labels) { + this.labels = labels; + } + + public String getParams() { + return params; + } + + public void setParams(String params) { + this.params = params; + } + + public String getProgress() { + return progress; + } + + public void setProgress(String progress) { + this.progress = progress; + } + + public String getStatus() { + return status; + } + + public void setStatus(String status) { + this.status = status; + } + + public String getLogPath() { + return logPath; + } + + public void setLogPath(String logPath) { + this.logPath = logPath; + } + + public Integer getErrorCode() { + return errorCode; + } + + public void setErrorCode(Integer errorCode) { + this.errorCode = errorCode; + } + + public String getErrorDesc() { + return errorDesc; + } + + public void setErrorDesc(String errorDesc) { + this.errorDesc = errorDesc; + } + + public Date getCreatedTime() { + return createdTime; + } + + public void setCreatedTime(Date createdTime) { + this.createdTime = createdTime; + } + + public Date getUpdatedTime() { + return updatedTime; + } + + public void setUpdatedTime(Date updatedTime) { + this.updatedTime = updatedTime; + } + + public String getInstances() { + return instances; + } + + public void setInstances(String instances) { + this.instances = instances; + } + + public String getMetrics() { + return metrics; + } + + public void setMetrics(String metrics) { + this.metrics = metrics; + } + + public String getEngineType() { + return engineType; + } + + public void setEngineType(String engineType) { + this.engineType = engineType; + } + + public String getExecutionCode() { + return executionCode; + } + + public void setExecutionCode(String executionCode) { + this.executionCode = executionCode; + } + + public String getUpdateTimeMills() { + return QueryUtils.dateToString(getUpdatedTime()); + } + + public String getObserveInfo() { + return observeInfo; + } + + public void setObserveInfo(String observeInfo) { + this.observeInfo = observeInfo; + } +} diff --git a/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/jobhistory/exception/AnomalyScannerException.java b/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/jobhistory/exception/AnomalyScannerException.java new file mode 100644 index 0000000000..0e3dd10d6b --- /dev/null +++ b/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/jobhistory/exception/AnomalyScannerException.java @@ -0,0 +1,37 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.monitor.scan.app.jobhistory.exception; + +import org.apache.linkis.common.exception.ErrorException; +import org.apache.linkis.common.exception.ExceptionLevel; + +public class AnomalyScannerException extends ErrorException { + public AnomalyScannerException(int errCode, String desc) { + super(errCode, desc); + } + + public AnomalyScannerException( + int errCode, String desc, String ip, int port, String serviceKind) { + super(errCode, desc, ip, port, serviceKind); + } + + @Override + public ExceptionLevel getLevel() { + return super.getLevel(); + } +} diff --git a/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/jobhistory/exception/DirtyDataCleanException.java b/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/jobhistory/exception/DirtyDataCleanException.java new file mode 100644 index 0000000000..93b6065f75 --- /dev/null +++ b/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/jobhistory/exception/DirtyDataCleanException.java @@ -0,0 +1,37 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.monitor.scan.app.jobhistory.exception; + +import org.apache.linkis.common.exception.ErrorException; +import org.apache.linkis.common.exception.ExceptionLevel; + +public class DirtyDataCleanException extends ErrorException { + public DirtyDataCleanException(int errCode, String desc) { + super(errCode, desc); + } + + public DirtyDataCleanException( + int errCode, String desc, String ip, int port, String serviceKind) { + super(errCode, desc, ip, port, serviceKind); + } + + @Override + public ExceptionLevel getLevel() { + return super.getLevel(); + } +} diff --git a/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/monitor/config/ListenerConfig.java b/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/monitor/config/ListenerConfig.java new file mode 100644 index 0000000000..fae16b63a3 --- /dev/null +++ b/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/monitor/config/ListenerConfig.java @@ -0,0 +1,46 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.monitor.scan.app.monitor.config; + +import org.apache.linkis.monitor.scan.app.monitor.until.HttpsUntils; +import org.apache.linkis.monitor.scan.app.monitor.until.ThreadUtils; +import org.apache.linkis.monitor.scan.utils.log.LogUtils; + +import org.springframework.context.annotation.Configuration; +import org.springframework.context.event.ContextClosedEvent; +import org.springframework.context.event.EventListener; + +import java.io.IOException; + +import org.slf4j.Logger; + +@Configuration +public class ListenerConfig { + + private static final Logger logger = LogUtils.stdOutLogger(); + + @EventListener + private void shutdownEntrance(ContextClosedEvent event) { + try { + ThreadUtils.executors.shutdown(); + HttpsUntils.client.close(); + } catch (IOException e) { + logger.error("ListenerConfig error msg {}", e.getMessage()); + } + } +} diff --git a/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/monitor/config/MonitorConfig.java b/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/monitor/config/MonitorConfig.java new file mode 100644 index 0000000000..fab9a5cbe7 --- /dev/null +++ b/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/monitor/config/MonitorConfig.java @@ -0,0 +1,71 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.monitor.scan.app.monitor.config; + +import org.apache.linkis.common.conf.CommonVars; +import org.apache.linkis.common.conf.Configuration; + +public class MonitorConfig { + + public static final String shellPath = Configuration.getLinkisHome() + "/admin/"; + + public static final CommonVars GATEWAY_URL = CommonVars.apply("wds.linkis.gateway.url"); + + public static final CommonVars USER_MODE_TIMEOUT = + CommonVars.apply("linkis.monitor.user.timeOut", 300); + public static final CommonVars USER_MODE_AUTHTOKEN = + CommonVars.apply("linkis.monitor.user.authToken","VALIDATOR-AUTH"); + public static final CommonVars USER_MODE_ENGINE = + CommonVars.apply("linkis.monitor.user.enginelist","[]"); + + public static final CommonVars ECM_TASK_MAJOR = + CommonVars.apply("linkis.monitor.ecmResourceTask.major", 0.03); + public static final CommonVars ECM_TASK_MINOR = + CommonVars.apply("linkis.monitor.ecmResourceTask.minor", 0.1); + public static final CommonVars ECM_TASK_IMURL = + CommonVars.apply("linkis.monitor.metrics.imsUrl"); + public static final CommonVars ECM_TASK_USER_AUTHKEY = + CommonVars.apply("linkis.monitor.metrics.userAuthKey"); + + public static final CommonVars JOB_HISTORY_TIME_EXCEED = + CommonVars.apply("linkis.monitor.jobhistory.id.timeExceed",0L); + + public static final CommonVars ENTRANCE_TASK_USERTOTAL = + CommonVars.apply("linkis.monitor.entranceTask.userTotalTask", 1000); + public static final CommonVars ENTRANCE_TASK_TOTAL_MAJOR = + CommonVars.apply("linkis.monitor.entranceTask.linkisTotalTaskMajor", 50000); + public static final CommonVars ENTRANCE_TASK_TOTAL_MINOR = + CommonVars.apply("linkis.monitor.entranceTask.linkisTotalTaskMinor", 10000); + public static final CommonVars ENTRANCE_TASK_USERLIST = + CommonVars.apply("linkis.monitor.entranceTask.userlist","[]"); + + public static final CommonVars SCHEDULED_CONFIG_NUM = + CommonVars.apply("linkis.monitor.scheduled.pool.cores.num", 10); + + public static final CommonVars SHELL_TIMEOUT = + CommonVars.apply("linkis.monitor.shell.time.out.minute", 30); + + public static final CommonVars USER_MODE_INTERFACE_TIMEOUT = + CommonVars.apply("linkis.monitor.user.mode.time.out", 30*1000); + + public static final CommonVars CHATBOT_KEY_ID = CommonVars.apply("linkis.monitor.chatbot.key.id","23e6afad1b78a0c5eed67e4d24de7063"); + public static final CommonVars CHATBOT_TYPE = CommonVars.apply("linkis.monitor.chatbot.type","text"); + public static final CommonVars CHATBOT_SERVICE_NAME= CommonVars.apply("linkis.monitor.chatbot.serviceName","大数据生产助手(BDP_PRD)"); + public static final CommonVars CHATBOT_URL= CommonVars.apply("linkis.monitor.chatbot.url","http://172.21.3.43:1377/pros-chatbot/yuanfang/sendEMsg"); + public static final CommonVars SOLUTION_URL = CommonVars.apply("linkis.monitor.jobhistory.solution.url", "http://kn.dss.weoa.com/linkis/qa"); +} diff --git a/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/monitor/config/ScheduledConfig.java b/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/monitor/config/ScheduledConfig.java new file mode 100644 index 0000000000..30495a87e1 --- /dev/null +++ b/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/monitor/config/ScheduledConfig.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.monitor.scan.app.monitor.config; + +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.scheduling.annotation.SchedulingConfigurer; +import org.springframework.scheduling.config.ScheduledTaskRegistrar; + +import java.util.concurrent.Executor; +import java.util.concurrent.Executors; + +@Configuration +public class ScheduledConfig implements SchedulingConfigurer { + @Bean + public Executor taskExecutor() { + return Executors.newScheduledThreadPool(MonitorConfig.SCHEDULED_CONFIG_NUM.getValue()); + } + + @Override + public void configureTasks(ScheduledTaskRegistrar scheduledTaskRegistrar) { + scheduledTaskRegistrar.setScheduler(taskExecutor()); + } +} diff --git a/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/monitor/entity/ChatbotEntity.java b/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/monitor/entity/ChatbotEntity.java new file mode 100644 index 0000000000..3cf288adbb --- /dev/null +++ b/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/monitor/entity/ChatbotEntity.java @@ -0,0 +1,87 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.monitor.scan.app.monitor.entity; + +import org.apache.linkis.monitor.scan.app.monitor.config.MonitorConfig; + +public class ChatbotEntity { + private String keyid; + private String content; + private String type; + private String userName; + private String serviceName; + + public ChatbotEntity(String content, String userName) { + this.keyid = MonitorConfig.CHATBOT_KEY_ID.getValue(); + this.content = content; + this.type = MonitorConfig.CHATBOT_TYPE.getValue(); + this.userName = userName; + this.serviceName = MonitorConfig.CHATBOT_SERVICE_NAME.getValue(); + } + + public String getKeyid() { + return keyid; + } + + public void setKeyid(String keyid) { + this.keyid = keyid; + } + + public String getContent() { + return content; + } + + public void setContent(String content) { + this.content = content; + } + + public String getType() { + return type; + } + + public void setType(String type) { + this.type = type; + } + + public String getUserName() { + return userName; + } + + public void setUserName(String userName) { + this.userName = userName; + } + + public String getServiceName() { + return serviceName; + } + + public void setServiceName(String serviceNameuserName) { + this.serviceName = serviceNameuserName; + } + + @Override + public String toString() { + return "ChatbotEntity{" + + "keyid='" + keyid + '\'' + + ", content='" + content + '\'' + + ", type='" + type + '\'' + + ", userName='" + userName + '\'' + + ", serviceName='" + serviceName + '\'' + + '}'; + } +} diff --git a/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/monitor/entity/EngineEntity.java b/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/monitor/entity/EngineEntity.java new file mode 100644 index 0000000000..650fa081cc --- /dev/null +++ b/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/monitor/entity/EngineEntity.java @@ -0,0 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.monitor.scan.app.monitor.entity; + +import java.io.Serializable; + +public class EngineEntity implements Serializable { + + private String engineType; + + private String code; + + private String runType; + + public String getEngineType() { + return engineType; + } + + public void setEngineType(String engineType) { + this.engineType = engineType; + } + + public String getCode() { + return code; + } + + public void setCode(String code) { + this.code = code; + } + + public String getRunType() { + return runType; + } + + public void setRunType(String runType) { + this.runType = runType; + } +} diff --git a/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/monitor/entity/EntranceEntity.java b/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/monitor/entity/EntranceEntity.java new file mode 100644 index 0000000000..6fa9441474 --- /dev/null +++ b/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/monitor/entity/EntranceEntity.java @@ -0,0 +1,63 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.monitor.scan.app.monitor.entity; + +import java.io.Serializable; + +public class EntranceEntity implements Serializable { + + private Integer runningtasks; + + private Integer queuedtasks; + + private String alteruser; + + private String username; + + public Integer getQueuedtasks() { + return queuedtasks; + } + + public void setQueuedtasks(Integer queuedtasks) { + this.queuedtasks = queuedtasks; + } + + public String getAlteruser() { + return alteruser; + } + + public void setAlteruser(String alteruser) { + this.alteruser = alteruser; + } + + public String getUsername() { + return username; + } + + public void setUsername(String username) { + this.username = username; + } + + public Integer getRunningtasks() { + return runningtasks; + } + + public void setRunningtasks(Integer runningtasks) { + this.runningtasks = runningtasks; + } +} diff --git a/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/monitor/entity/IndexEntity.java b/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/monitor/entity/IndexEntity.java new file mode 100644 index 0000000000..fbea8886e8 --- /dev/null +++ b/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/monitor/entity/IndexEntity.java @@ -0,0 +1,85 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.monitor.scan.app.monitor.entity; + +import org.apache.linkis.monitor.scan.constants.Constants; + +public class IndexEntity { + + private final String subsystemId = Constants.ALERT_SUB_SYSTEM_ID(); + private String interfaceName; + private String attrGroup; + private String attrName; + private String hostIp; + private String metricValue; + + public String getSubsystemId() { + return subsystemId; + } + + public String getInterfaceName() { + return interfaceName; + } + + public void setInterfaceName(String interfaceName) { + this.interfaceName = interfaceName; + } + + public String getAttrGroup() { + return attrGroup; + } + + public void setAttrGroup(String attrGroup) { + this.attrGroup = attrGroup; + } + + public String getAttrName() { + return attrName; + } + + public void setAttrName(String attrName) { + this.attrName = attrName; + } + + public String getHostIp() { + return hostIp; + } + + public void setHostIp(String hostIp) { + this.hostIp = hostIp; + } + + public String getMetricValue() { + return metricValue; + } + + public void setMetricValue(String metricValue) { + this.metricValue = metricValue; + } + + public IndexEntity() {} + + public IndexEntity( + String interfaceName, String attrGroup, String attrName, String hostIp, String metricValue) { + this.interfaceName = interfaceName; + this.attrGroup = attrGroup; + this.attrName = attrName; + this.hostIp = hostIp; + this.metricValue = metricValue; + } +} diff --git a/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/monitor/scheduled/BmlClear.java b/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/monitor/scheduled/BmlClear.java new file mode 100644 index 0000000000..75d415ac0d --- /dev/null +++ b/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/monitor/scheduled/BmlClear.java @@ -0,0 +1,47 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.monitor.scan.app.monitor.scheduled; + +import org.apache.linkis.bml.cleaner.service.CleanerService; +import org.apache.linkis.monitor.scan.utils.log.LogUtils; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.annotation.PropertySource; +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.stereotype.Component; + +import java.util.*; + +import org.slf4j.Logger; + + +@Component +@PropertySource(value = "classpath:linkis-et-monitor.properties", encoding = "UTF-8") +public class BmlClear { + + private static final Logger logger = LogUtils.stdOutLogger(); + + @Autowired private CleanerService cleanerServices; + + @Scheduled(cron = "${linkis.monitor.bml.clear.historyVersion.cron}") + public void jobHistoryScanTask() { + logger.info("start to clear bml history version"); + cleanerServices.run(); + logger.info("end to clear bml history version"); + } +} diff --git a/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/monitor/scheduled/EcRecordClear.java b/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/monitor/scheduled/EcRecordClear.java new file mode 100644 index 0000000000..dced6117b1 --- /dev/null +++ b/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/monitor/scheduled/EcRecordClear.java @@ -0,0 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.monitor.scan.app.monitor.scheduled; + +import org.apache.linkis.monitor.scan.app.monitor.config.MonitorConfig; +import org.apache.linkis.monitor.scan.app.monitor.until.ThreadUtils; +import org.apache.linkis.monitor.scan.utils.log.LogUtils; + +import org.springframework.context.annotation.PropertySource; +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.stereotype.Component; + +import java.util.ArrayList; +import java.util.List; + +import org.slf4j.Logger; + +/*** + * Task: clean up linkis_cg_ec_resource_info_record data + */ +@Component +@PropertySource(value = "classpath:linkis-et-monitor.properties", encoding = "UTF-8") +public class EcRecordClear { + + private static final Logger logger = LogUtils.stdOutLogger(); + + @Scheduled(cron = "${linkis.monitor.clear.ecRecord.cron}") + public void ecRecordClear() { + logger.info("Start to clear_ec_record shell"); + List cmdlist = new ArrayList<>(); + cmdlist.add("sh"); + cmdlist.add(MonitorConfig.shellPath + "clear_ec_record.sh"); + logger.info("clear_ec_record shell command {}", cmdlist); + String exec = ThreadUtils.run(cmdlist, "clear_ec_record.sh"); + logger.info("shell log {}", exec); + logger.info("End to clear_ec_record shell "); + } +} diff --git a/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/monitor/scheduled/JobHistoryClear.java b/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/monitor/scheduled/JobHistoryClear.java new file mode 100644 index 0000000000..4880a600cc --- /dev/null +++ b/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/monitor/scheduled/JobHistoryClear.java @@ -0,0 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.monitor.scan.app.monitor.scheduled; + +import org.apache.linkis.monitor.scan.app.monitor.config.MonitorConfig; +import org.apache.linkis.monitor.scan.app.monitor.until.ThreadUtils; +import org.apache.linkis.monitor.scan.utils.log.LogUtils; + +import org.springframework.context.annotation.PropertySource; +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.stereotype.Component; + +import java.util.ArrayList; +import java.util.List; + +import org.slf4j.Logger; + +/*** + * Task: clean up linkis_ps_job_history_group_history data + */ +@Component +@PropertySource(value = "classpath:linkis-et-monitor.properties", encoding = "UTF-8") +public class JobHistoryClear { + + private static final Logger logger = LogUtils.stdOutLogger(); + + @Scheduled(cron = "${linkis.monitor.clear.historyTask.cron}") + public void historyTaskClear() { + logger.info("Start to clear_history_task shell"); + List cmdlist = new ArrayList<>(); + cmdlist.add("sh"); + cmdlist.add(MonitorConfig.shellPath + "clear_history_task.sh"); + logger.info("clear_history_task shell command {}", cmdlist); + String exec = ThreadUtils.run(cmdlist, "clear_history_task.sh"); + logger.info("shell log {}", exec); + logger.info("End to clear_history_task shell "); + } +} diff --git a/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/monitor/scheduled/JobHistoryMonitor.java b/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/monitor/scheduled/JobHistoryMonitor.java new file mode 100644 index 0000000000..01873a988b --- /dev/null +++ b/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/monitor/scheduled/JobHistoryMonitor.java @@ -0,0 +1,264 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.monitor.scan.app.monitor.scheduled; + +import org.apache.linkis.monitor.scan.app.factory.MapperFactory; +import org.apache.linkis.monitor.scan.app.jobhistory.JobHistoryDataFetcher; +import org.apache.linkis.monitor.scan.app.jobhistory.errorcode.JobHistoryErrCodeRule; +import org.apache.linkis.monitor.scan.app.jobhistory.errorcode.JobHistoryErrorCodeAlertSender; +import org.apache.linkis.monitor.scan.app.jobhistory.jobtime.JobTimeExceedAlertSender; +import org.apache.linkis.monitor.scan.app.jobhistory.jobtime.JobTimeExceedRule; +import org.apache.linkis.monitor.scan.app.jobhistory.labels.JobHistoryLabelsAlertSender; +import org.apache.linkis.monitor.scan.app.jobhistory.labels.JobHistoryLabelsRule; +import org.apache.linkis.monitor.scan.app.jobhistory.runtime.CommonRunTimeAlertSender; +import org.apache.linkis.monitor.scan.app.jobhistory.runtime.CommonJobRunTimeRule; +import org.apache.linkis.monitor.scan.app.jobhistory.runtime.JobHistoryRunTimeAlertSender; +import org.apache.linkis.monitor.scan.app.jobhistory.runtime.JobHistoryRunTimeRule; +import org.apache.linkis.monitor.scan.app.monitor.config.MonitorConfig; +import org.apache.linkis.monitor.scan.app.monitor.until.CacheUtils; +import org.apache.linkis.monitor.scan.constants.Constants; +import org.apache.linkis.monitor.scan.core.pac.DataFetcher; +import org.apache.linkis.monitor.scan.core.scanner.AnomalyScanner; +import org.apache.linkis.monitor.scan.core.scanner.DefaultScanner; +import org.apache.linkis.monitor.scan.utils.alert.AlertDesc; +import org.apache.linkis.monitor.scan.utils.alert.ims.ImsAlertDesc; +import org.apache.linkis.monitor.scan.utils.alert.ims.JobHistoryScanImsAlertPropFileParserUtils; +import org.apache.linkis.monitor.scan.utils.alert.ims.UserLabelAlertUtils; +import org.apache.linkis.monitor.scan.utils.log.LogUtils; + +import org.springframework.context.annotation.PropertySource; +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.stereotype.Component; + +import java.util.*; + +import org.slf4j.Logger; + +/*** + * jobHistory monitor + * 1.errorCode: Monitor the error code + * 2.userLabel: tenant label monitoring, scan the execution data within the first 20 minutes, and judge the labels field of the data + * 3.jobResultRunTime: Scan the execution data within the first 20 minutes, and judge the completed tasks. If the parm field in the jobhistory contains (task.notification.conditions) and the result of executing the task is (Succeed, Failed, Canceled, Timeout, ALL) any one of them, an alarm is triggered and the result of the job is that it has ended. All three are indispensable + * 4.jobResultRunTimeForDSS: Scan the execution data within the first 20 minutes, scan the tasks that have been marked for notification, if the task has ended, a notification will be initiated + * 5.jobHistoryUnfinishedScan: monitor the status of the execution task, scan the data outside 12 hours and within 24 hours + */ +@Component +@PropertySource(value = "classpath:linkis-et-monitor.properties", encoding = "UTF-8") +public class JobHistoryMonitor { + + private static final Logger logger = LogUtils.stdOutLogger(); + private static final long backtrackNum = 1000000L; + + /** + * Scan tasks completed within 20 minutes + */ + @Scheduled(cron = "0 0/10 * * * ?") + public void jobHistoryFinishedScan() { + long intervalMs = 20 * 60 * 1000; // 20分钟 + long maxIntervalMs = Constants.MAX_INTERVALS_SECONDS() * 1000; + long endTime = System.currentTimeMillis(); + long startTime = endTime - intervalMs; + long realIntervals = endTime - startTime < maxIntervalMs ? endTime - startTime : maxIntervalMs; + AnomalyScanner scanner = new DefaultScanner(); + boolean shouldStart = false; + long id; + if (null == CacheUtils.cacheBuilder.getIfPresent("jobHistoryId")) { + //20230206：新增获取最大值-100W 作为初始id进行查询，防止慢查询 + long maxId = MapperFactory.getJobHistoryMapper().selectMaxId(); + long beginId = 0L; + if (maxId > backtrackNum) { + beginId = maxId - backtrackNum; + } + id = MapperFactory.getJobHistoryMapper().selectIdByHalfDay(beginId); + CacheUtils.cacheBuilder.put("jobHistoryId", id); + } else { + id = CacheUtils.cacheBuilder.getIfPresent("jobHistoryId"); + } + List fetchers = generateFetchersfortime(startTime, endTime, id, "updated_time"); + if (fetchers == null) { + logger.warn("generated 0 dataFetchers, plz check input"); + return; + } + // errorCode + try { + Map errorCodeAlerts = JobHistoryScanImsAlertPropFileParserUtils.getAlerts(Constants.SCAN_PREFIX_ERRORCODE()); + + if (errorCodeAlerts == null || errorCodeAlerts.size() == 0) { + logger.info("[INFO] Loaded 0 errorcode alert from alert-rule properties file."); + } else { + logger.info( + "[INFO] Loaded {} errorcode alert from alert-rules properties file.", + errorCodeAlerts.size()); + shouldStart = true; + addIntervalToImsAlerts(errorCodeAlerts, realIntervals); + JobHistoryErrCodeRule jobHistoryErrCodeRule = + new JobHistoryErrCodeRule( + errorCodeAlerts.keySet(), new JobHistoryErrorCodeAlertSender(errorCodeAlerts)); + scanner.addScanRule(jobHistoryErrCodeRule); + } + } catch (Exception e) { + logger.warn("Jobhistory Monitor ErrorCode Faily: "+ e.getMessage()); + } + // userLabel + try { + Map userLabelAlerts = + UserLabelAlertUtils.getAlerts(Constants.USER_LABEL_MONITOR(), ""); + if (userLabelAlerts == null || userLabelAlerts.size() == 0) { + logger.info("[INFO] Loaded 0 alerts userLabel alert-rule from alert properties file."); + } else { + logger.info( + "[INFO] Loaded {} alerts userLabel alert-rules from alert properties file.", + userLabelAlerts.size()); + shouldStart = true; + JobHistoryLabelsRule jobHistoryLabelsRule = + new JobHistoryLabelsRule(new JobHistoryLabelsAlertSender()); + scanner.addScanRule(jobHistoryLabelsRule); + } + } catch (Exception e) { + logger.warn("Jobhistory Monitor UserLabel Faily: "+ e.getMessage()); + } + // jobResultRunTime + try { + Map jobResultAlerts = + JobHistoryScanImsAlertPropFileParserUtils.getAlerts(Constants.SCAN_PREFIX_ERRORCODE()); + if (jobResultAlerts == null || jobResultAlerts.size() == 0) { + logger.info("[INFO] Loaded 0 jobResult alert from alert-rule properties file."); + } else { + logger.info( + "[INFO] Loaded {} alerts jobResult alert-rules from alert properties file.", + jobResultAlerts.size()); + shouldStart = true; + JobHistoryRunTimeRule jobHistoryRunTimeRule = + new JobHistoryRunTimeRule(new JobHistoryRunTimeAlertSender()); + scanner.addScanRule(jobHistoryRunTimeRule); + } + } catch (Exception e) { + logger.warn("Jobhistory Monitor JobResultRunTime Faily: "+ e.getMessage()); + } + // jobResultRunTimeForDSS + try { + Map dssJobResultAlerts = + JobHistoryScanImsAlertPropFileParserUtils.getAlerts(Constants.SCAN_PREFIX_ERRORCODE()); + if (dssJobResultAlerts == null || dssJobResultAlerts.size() == 0) { + logger.info("[INFO] Loaded 0 jobResult alert from alert-rule properties file."); + } else { + logger.info( + "[INFO] Loaded {} alerts jobResult alert-rules from alert properties file.", + dssJobResultAlerts.size()); + shouldStart = true; + + CommonJobRunTimeRule commonJobRunTimeRule = + new CommonJobRunTimeRule(new CommonRunTimeAlertSender()); + scanner.addScanRule(commonJobRunTimeRule); + } + } catch (Exception e) { + logger.warn("Jobhistory JobResultRunTimeForDSS ErrorCode Faily: "+ e.getMessage()); + } + run(scanner, fetchers, shouldStart); + } + + /*** + * Whether scanning data within 12 hours has timed out + */ + @Scheduled(cron = "${linkis.monitor.jobHistory.ScanTask.cron}") + public void jobHistoryUnfinishedScan() { + long id = + Optional.ofNullable(CacheUtils.cacheBuilder.getIfPresent("jobhistoryScan")) + .orElse(MonitorConfig.JOB_HISTORY_TIME_EXCEED.getValue()); + long intervalMs = Constants.SCAN_INTERVALS_SECONDS() * 1000; + long maxIntervalMs = Constants.MAX_INTERVALS_SECONDS() * 1000; + long endTime = System.currentTimeMillis(); + long startTime = endTime - intervalMs; + long realIntervals = endTime - startTime < maxIntervalMs ? endTime - startTime : maxIntervalMs; + AnomalyScanner scanner = new DefaultScanner(); + boolean shouldStart = false; + List fetchers = generateFetchers(startTime, endTime, maxIntervalMs, id, "created_time"); + if (fetchers == null) { + logger.warn("generated 0 dataFetchers, plz check input"); + return; + } + Map jobTimeAlerts = + JobHistoryScanImsAlertPropFileParserUtils.getAlerts( + Constants.SCAN_PREFIX_UNFINISHED_JOBTIME_EXCEED_SEC()); + if (jobTimeAlerts == null || jobTimeAlerts.size() == 0) { + logger.info("[INFO] Loaded 0 alerts jobtime alert-rule from alert properties file."); + } else { + logger.info( + "[INFO] Loaded {} alerts jobtime alert-rules from alert properties file.", + jobTimeAlerts.size()); + shouldStart = true; + addIntervalToImsAlerts(jobTimeAlerts, realIntervals); + JobTimeExceedRule jobTimeExceedRule = + new JobTimeExceedRule( + jobTimeAlerts.keySet(), new JobTimeExceedAlertSender(jobTimeAlerts)); + scanner.addScanRule(jobTimeExceedRule); + } + run(scanner, fetchers, shouldStart); + } + + public static void run(AnomalyScanner scanner, List fetchers, Boolean shouldStart) { + if (shouldStart) { + scanner.addDataFetchers(fetchers); + scanner.run(); + // scanner.shutdown(); // wait all alert to be send + } + } + + private static List generateFetchers( + long startTime, long endTime, long maxIntervalMs, long id, String timeType) { + List ret = new ArrayList<>(); + long pe = endTime; + long ps; + while (pe > startTime) { + ps = pe - maxIntervalMs > startTime ? pe - maxIntervalMs : startTime; + String[] fetcherArgs = + new String[]{String.valueOf(ps), String.valueOf(pe), String.valueOf(id), timeType}; + ret.add(new JobHistoryDataFetcher(fetcherArgs, MapperFactory.getJobHistoryMapper())); + logger.info( + "Generated dataFetcher for startTime: " + + new Date(ps).toString() + + ". EndTime: " + + new Date(pe).toString()); + pe = pe - maxIntervalMs; + } + return ret; + } + + private static List generateFetchersfortime(long startTime, long endTime, long id, String timeType) { + List fetchers = new ArrayList<>(); + String[] fetcherArgs = + new String[]{String.valueOf(startTime), String.valueOf(endTime), String.valueOf(id), timeType}; + fetchers.add(new JobHistoryDataFetcher(fetcherArgs, MapperFactory.getJobHistoryMapper())); + logger.info( + "Generated dataFetcher for startTime: " + + new Date(startTime).toString() + + ". EndTime: " + + new Date(endTime).toString()); + return fetchers; + } + + private static void addIntervalToImsAlerts(Map alerts, long realIntervals) { + for (AlertDesc alert : alerts.values()) { + if (!(alert instanceof ImsAlertDesc)) { + logger.info("[warn] ignore wrong alert" + alert); + } else { + ((ImsAlertDesc) alert).hitIntervalMs_$eq(realIntervals); + } + } + } +} diff --git a/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/monitor/scheduled/ResourceMonitor.java b/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/monitor/scheduled/ResourceMonitor.java new file mode 100644 index 0000000000..b7066ba420 --- /dev/null +++ b/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/monitor/scheduled/ResourceMonitor.java @@ -0,0 +1,151 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.monitor.scan.app.monitor.scheduled; + +import org.apache.linkis.common.utils.ByteTimeUtils; +import org.apache.linkis.monitor.scan.app.monitor.config.MonitorConfig; +import org.apache.linkis.monitor.scan.app.monitor.entity.IndexEntity; +import org.apache.linkis.monitor.scan.app.monitor.until.HttpsUntils; +import org.apache.linkis.monitor.scan.constants.Constants; +import org.apache.linkis.monitor.scan.utils.alert.AlertDesc; +import org.apache.linkis.monitor.scan.utils.alert.ims.MonitorAlertUtils; +import org.apache.linkis.monitor.scan.utils.alert.ims.PooledImsAlertUtils; + +import org.apache.commons.collections.MapUtils; +import org.apache.commons.lang3.StringUtils; + +import org.springframework.context.annotation.PropertySource; +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.stereotype.Component; + +import java.io.IOException; +import java.math.BigDecimal; +import java.util.*; +import java.util.concurrent.atomic.AtomicReference; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/*** + * Monitor the usage of ECM resources for monitoring and metrics reporting + */ +@Component +@PropertySource(value = "classpath:linkis-et-monitor.properties", encoding = "UTF-8") +public class ResourceMonitor { + + private static final Logger logger = LoggerFactory.getLogger(ResourceMonitor.class); + + @Scheduled(cron = "${linkis.monitor.ecmResourceTask.cron}") + public void ecmResourceTask() { + Map resultmap = null; + AtomicReference tenant = new AtomicReference<>("租户标签：公共资源"); + AtomicReference totalMemory = new AtomicReference<>(0.0); + AtomicReference totalInstance = new AtomicReference<>(0.0); + AtomicReference totalCores = new AtomicReference<>(0.0); + try { + resultmap = HttpsUntils.sendHttp(null, null); + logger.info("ResourceMonitor response {}:", resultmap); + } catch (IOException e) { + logger.warn("failed to get EcmResource data"); + } + // got interface data + Map>> data = MapUtils.getMap(resultmap, "data"); + List> emNodeVoList = data.getOrDefault("EMs", new ArrayList<>()); + StringJoiner minor = new StringJoiner(","); + StringJoiner major = new StringJoiner(","); + // deal ecm resource + emNodeVoList.forEach( + emNodeVo -> { + Map leftResource = MapUtils.getMap(emNodeVo, "leftResource"); + Map maxResource = MapUtils.getMap(emNodeVo, "maxResource"); + // 新增 ECM资源告警，需补充此ECM所属租户 + List> labels = (List>) emNodeVo.get("labels"); + labels.stream() + .filter(labelmap -> labelmap.containsKey("tenant")) + .forEach(map -> tenant.set("租户标签：" + map.get("stringValue").toString())); + String leftmemory = ByteTimeUtils.bytesToString((long) leftResource.getOrDefault("memory",0)); + String maxmemory = ByteTimeUtils.bytesToString((long) maxResource.getOrDefault("memory",0)); + + String leftmemoryStr = leftmemory.split(" ")[0]; + String maxmemoryStr = maxmemory.split(" ")[0]; + + BigDecimal leftMemory = new BigDecimal(leftmemoryStr); + BigDecimal leftCores = new BigDecimal((int) leftResource.get("cores")); + BigDecimal leftInstance = new BigDecimal((int) leftResource.get("instance")); + totalMemory.set(totalMemory.get() + leftMemory.doubleValue()); + totalInstance.set(totalInstance.get() + leftInstance.doubleValue()); + totalCores.set(totalCores.get() + leftCores.doubleValue()); + + BigDecimal maxMemory = new BigDecimal(maxmemoryStr); + BigDecimal maxCores = new BigDecimal((int) maxResource.get("cores")); + BigDecimal maxInstance = new BigDecimal((int) maxResource.get("instance")); + double memorydouble = leftMemory.divide(maxMemory,2, BigDecimal.ROUND_HALF_DOWN).doubleValue(); + double coresdouble = leftCores.divide(maxCores,2, BigDecimal.ROUND_HALF_DOWN).doubleValue(); + double instancedouble = leftInstance.divide(maxInstance,2, BigDecimal.ROUND_HALF_DOWN).doubleValue(); + Double majorValue = MonitorConfig.ECM_TASK_MAJOR.getValue(); + Double minorValue = MonitorConfig.ECM_TASK_MINOR.getValue(); + if (((memorydouble) <= majorValue) + || ((coresdouble) <= majorValue) + || ((instancedouble) <= majorValue)) { + // major告警 + major.add(emNodeVo.get("instance").toString()); + } else if (((memorydouble) < minorValue) + || ((coresdouble) < minorValue) + || ((instancedouble) < minorValue)) { + // minor告警 + minor.add(emNodeVo.get("instance").toString()); + } + HashMap replaceParm = new HashMap<>(); + replaceParm.put("$tenant", tenant.get()); + if (StringUtils.isNotBlank(major.toString())) { + replaceParm.put("$instance", major.toString()); + replaceParm.put("$ratio", majorValue.toString()); + Map ecmResourceAlerts = + MonitorAlertUtils.getAlerts(Constants.ALERT_RESOURCE_MONITOR(), replaceParm); + PooledImsAlertUtils.addAlert(ecmResourceAlerts.get("12004")); + } + if (StringUtils.isNotBlank(minor.toString())) { + replaceParm.put("$instance", minor.toString()); + replaceParm.put("$ratio", minorValue.toString()); + Map ecmResourceAlerts = + MonitorAlertUtils.getAlerts(Constants.ALERT_RESOURCE_MONITOR(), replaceParm); + PooledImsAlertUtils.addAlert(ecmResourceAlerts.get("12003")); + } + // ECM资源占比上报 + resourceSendToIms(coresdouble, memorydouble, instancedouble, HttpsUntils.localHost,"USED"); + }); + //ECM 剩余资源总数上报 + resourceSendToIms(totalCores.get(), totalMemory.get(), totalInstance.get(), HttpsUntils.localHost,"TOTAL"); + } + + private void resourceSendToIms(Double coresdouble, Double memorydouble, Double instancedouble, String loaclhost, String name) { + List list = new ArrayList<>(); + logger.info("ResourceMonitor send index "); + String core ="ECM_CPU_"; + String memory ="ECM_MEMORY_"; + String instance ="ECM_INSTANCE_"; + list.add(new IndexEntity(core.concat(name), "CPU", "INDEX", loaclhost, String.valueOf(coresdouble))); + list.add(new IndexEntity(memory.concat(name), "MEMORY", "INDEX", loaclhost, String.valueOf(memorydouble))); + list.add(new IndexEntity(instance.concat(name), "INSTANCE", "INDEX", loaclhost, String.valueOf(instancedouble))); + try { + HttpsUntils.sendIndex(list); + } catch (IOException e) { + logger.warn("failed to send EcmResource index"); + } + } +} diff --git a/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/monitor/scheduled/TaskLogClear.java b/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/monitor/scheduled/TaskLogClear.java new file mode 100644 index 0000000000..bae3dc53f7 --- /dev/null +++ b/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/monitor/scheduled/TaskLogClear.java @@ -0,0 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.monitor.scan.app.monitor.scheduled; + +import org.apache.linkis.monitor.scan.app.monitor.config.MonitorConfig; +import org.apache.linkis.monitor.scan.app.monitor.until.ThreadUtils; +import org.apache.linkis.monitor.scan.utils.log.LogUtils; + +import org.springframework.context.annotation.PropertySource; +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.stereotype.Component; + +import java.util.ArrayList; +import java.util.List; + +import org.slf4j.Logger; + +/*** + * Task: clean up logs, file data of ec materials + */ +@Component +@PropertySource(value = "classpath:linkis-et-monitor.properties", encoding = "UTF-8") +public class TaskLogClear { + + private static final Logger logger = LogUtils.stdOutLogger(); + + @Scheduled(cron = "${linkis.monitor.clear.taskLog.cron}") + public void taskLogClear() { + logger.info("Start to linkis_task_res_log_clear shell"); + List cmdlist = new ArrayList<>(); + cmdlist.add("sh"); + cmdlist.add(MonitorConfig.shellPath + "linkis_task_res_log_clear.sh"); + logger.info("linkis_task_res_log_clear shell command {}", cmdlist); + String exec = ThreadUtils.run(cmdlist, "linkis_task_res_log_clear.sh"); + logger.info("shell log {}", exec); + logger.info("End to linkis_task_res_log_clear shell "); + } +} diff --git a/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/monitor/scheduled/TaskMonitor.java b/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/monitor/scheduled/TaskMonitor.java new file mode 100644 index 0000000000..87bc6d6e22 --- /dev/null +++ b/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/monitor/scheduled/TaskMonitor.java @@ -0,0 +1,183 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.monitor.scan.app.monitor.scheduled; + +import org.apache.linkis.common.ServiceInstance; +import org.apache.linkis.monitor.scan.app.monitor.config.MonitorConfig; +import org.apache.linkis.monitor.scan.app.monitor.entity.IndexEntity; +import org.apache.linkis.monitor.scan.app.monitor.until.HttpsUntils; +import org.apache.linkis.monitor.scan.constants.Constants; +import org.apache.linkis.monitor.scan.utils.alert.AlertDesc; +import org.apache.linkis.monitor.scan.utils.alert.ims.MonitorAlertUtils; +import org.apache.linkis.monitor.scan.utils.alert.ims.PooledImsAlertUtils; +import org.apache.linkis.rpc.Sender; +import org.apache.linkis.server.BDPJettyServerHelper; + +import org.apache.commons.collections.MapUtils; +import org.apache.commons.lang3.StringUtils; + +import org.springframework.context.annotation.PropertySource; +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.stereotype.Component; + +import java.io.IOException; +import java.math.BigDecimal; +import java.util.*; + +import com.google.gson.internal.LinkedTreeMap; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/*** + * Entrance monitors the number of tasks for specified users and systems. + * If the configured threshold is exceeded, an alarm will be triggered. + */ +@Component +@PropertySource(value = "classpath:linkis-et-monitor.properties", encoding = "UTF-8") +public class TaskMonitor { + + private static final Logger logger = LoggerFactory.getLogger(TaskMonitor.class); + + private static final String ENTRANCE_RUNNING_TASK = "entrance_running_task"; + private static final String ENTRANCE_QUEUED_TASK = "entrance_queued_task"; + + + @Scheduled(cron = "${linkis.monitor.entranceTask.cron}") + public void entranceTask() throws IOException { + List> userlist = new ArrayList<>(); + String value = MonitorConfig.ENTRANCE_TASK_USERLIST.getValue(); + if (StringUtils.isNotBlank(value)) { + userlist = BDPJettyServerHelper.gson().fromJson(value, ArrayList.class); + } + // 用户监控 + userlist.forEach( + entranceEntity -> { + Map data = new HashMap<>(); + try { + data = + MapUtils.getMap( + HttpsUntils.getEntranceTask(null, entranceEntity.get("username"),null), "data"); + logger.info("TaskMonitor userlist response {}:", data); + } catch (IOException e) { + logger.warn("failed to get EntranceTask data"); + } + + int runningNumber = + null != entranceEntity.get("runningtasks") + ? Integer.parseInt(entranceEntity.get("runningtasks")) + : 0; + int queuedNumber = + null != entranceEntity.get("queuedtasks") + ? Integer.parseInt(entranceEntity.get("queuedtasks")) + : 0; + + BigDecimal runningtotal = new BigDecimal((int) data.get("runningNumber")); + BigDecimal queuedtotal = new BigDecimal((int) data.get("queuedNumber")); + BigDecimal total = runningtotal.add(queuedtotal); + HashMap parms = new HashMap<>(); + parms.put("$username", entranceEntity.get("username")); + parms.put("$alteruser", entranceEntity.get("alteruser")); + parms.put("$url", MonitorConfig.GATEWAY_URL.getValue()); + // 获取标准阈值 + if (runningtotal.intValue() > runningNumber) { + // 触发告警 用户运行任务满 + parms.put("$runningtask", String.valueOf(runningNumber)); + Map ecmResourceAlerts = + MonitorAlertUtils.getAlerts(Constants.ALERT_RESOURCE_MONITOR(), parms); + PooledImsAlertUtils.addAlert(ecmResourceAlerts.get("12005")); + } + if (queuedtotal.intValue() > queuedNumber) { + // 触发告警 用户排队任务满 + parms.put("$queuedtask", String.valueOf(queuedNumber)); + Map ecmResourceAlerts = + MonitorAlertUtils.getAlerts(Constants.ALERT_RESOURCE_MONITOR(), parms); + PooledImsAlertUtils.addAlert(ecmResourceAlerts.get("12006")); + } + int usertotalTask = MonitorConfig.ENTRANCE_TASK_USERTOTAL.getValue(); + if (total.intValue() > usertotalTask) { + // 触发告警 用户任务总数满 + parms.put("$tasktotal", String.valueOf(usertotalTask)); + Map ecmResourceAlerts = + MonitorAlertUtils.getAlerts(Constants.ALERT_RESOURCE_MONITOR(), parms); + PooledImsAlertUtils.addAlert(ecmResourceAlerts.get("12007")); + } + }); + Map likisData = null; + try { + likisData = MapUtils.getMap(HttpsUntils.getEntranceTask(null, "hadoop",null), "data"); + logger.info("TaskMonitor hadoop response {}:", likisData); + } catch (IOException e) { + logger.warn("failed to get EntranceTask data"); + } + // 系统监控 + BigDecimal runningNumber = new BigDecimal((int) likisData.get("runningNumber")); + BigDecimal queuedNumber = new BigDecimal((int) likisData.get("queuedNumber")); + BigDecimal total = runningNumber.add(queuedNumber); + + HashMap parms = new HashMap<>(); + parms.put("$url", MonitorConfig.GATEWAY_URL.getValue()); + int linkisTotalMajor = MonitorConfig.ENTRANCE_TASK_TOTAL_MAJOR.getValue(); + int linkisTotalMinor = MonitorConfig.ENTRANCE_TASK_TOTAL_MINOR.getValue(); + if (total.intValue() >= linkisTotalMajor) { + // 触发告警Major + parms.put("$taskmajor", String.valueOf(linkisTotalMajor)); + logger.info("TaskMonitor parms {}:", parms); + Map ecmResourceAlerts = + MonitorAlertUtils.getAlerts(Constants.ALERT_RESOURCE_MONITOR(), parms); + PooledImsAlertUtils.addAlert(ecmResourceAlerts.get("12009")); + + } else if (total.intValue() >= linkisTotalMinor) { + // 触发告警Minor + parms.put("$taskminor", String.valueOf(linkisTotalMinor)); + logger.info("TaskMonitor parms {}:", parms); + Map ecmResourceAlerts = + MonitorAlertUtils.getAlerts(Constants.ALERT_RESOURCE_MONITOR(), parms); + PooledImsAlertUtils.addAlert(ecmResourceAlerts.get("12008")); + } + //指标上报 + resourceSendToIms(); + } + + public static void resourceSendToIms() { + //获取所有的entrance实例，逐个上送IMS + ServiceInstance[] instances = Sender.getInstances(Constants.DIRTY_DATA_ENTRANCE_APPLICATIONNAME()); + if (null != instances) { + for (ServiceInstance instance : instances) { + String serviceInstance = instance.getInstance(); + try { + Map instanceData = MapUtils.getMap(HttpsUntils.getEntranceTask(null, "hadoop", serviceInstance), "data"); + int runningNumber = 0; + int queuedNumber = 0; + if (instanceData.containsKey("runningNumber")) { + runningNumber = (int) instanceData.get("runningNumber"); + } + if (instanceData.containsKey("queuedNumber")) { + queuedNumber = (int) instanceData.get("queuedNumber"); + } + logger.info("ResourceMonitor send index "); + List list = new ArrayList<>(); + list.add(new IndexEntity(serviceInstance, "entrance", ENTRANCE_RUNNING_TASK, HttpsUntils.localHost, String.valueOf(runningNumber))); + list.add(new IndexEntity(serviceInstance, "entrance", ENTRANCE_QUEUED_TASK, HttpsUntils.localHost, String.valueOf(queuedNumber))); + HttpsUntils.sendIndex(list); + } catch (IOException e) { + logger.warn("failed to send EcmResource index :" + e); + } + } + } + } +} diff --git a/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/monitor/scheduled/UserModeMonitor.java b/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/monitor/scheduled/UserModeMonitor.java new file mode 100644 index 0000000000..03db536344 --- /dev/null +++ b/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/monitor/scheduled/UserModeMonitor.java @@ -0,0 +1,156 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.monitor.scan.app.monitor.scheduled; + +import org.apache.linkis.common.utils.Utils; +import org.apache.linkis.governance.common.entity.task.RequestPersistTask; +import org.apache.linkis.httpclient.dws.config.DWSClientConfig; +import org.apache.linkis.manager.label.constant.LabelKeyConstant; +import org.apache.linkis.monitor.scan.app.monitor.config.MonitorConfig; +import org.apache.linkis.monitor.scan.app.monitor.until.HttpsUntils; +import org.apache.linkis.monitor.scan.constants.Constants; +import org.apache.linkis.monitor.scan.utils.alert.AlertDesc; +import org.apache.linkis.monitor.scan.utils.alert.ims.MonitorAlertUtils; +import org.apache.linkis.monitor.scan.utils.alert.ims.PooledImsAlertUtils; +import org.apache.linkis.server.BDPJettyServerHelper; +import org.apache.linkis.ujes.client.UJESClient; +import org.apache.linkis.ujes.client.UJESClientImpl; +import org.apache.linkis.ujes.client.request.GetTableStatisticInfoAction; +import org.apache.linkis.ujes.client.request.JobSubmitAction; +import org.apache.linkis.ujes.client.response.GetTableStatisticInfoResult; +import org.apache.linkis.ujes.client.response.JobExecuteResult; +import org.apache.linkis.ujes.client.response.JobInfoResult; +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.stereotype.Component; +import java.net.SocketTimeoutException; +import java.util.*; +import java.util.concurrent.TimeUnit; + +import com.google.gson.internal.LinkedTreeMap; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/*** + * User mode monitoring: regularly trigger scripts to monitor whether the engine status is running normally + */ +@Component +public class UserModeMonitor { + + private static final Logger logger = LoggerFactory.getLogger(UserModeMonitor.class); + + private static final DWSClientConfig clientConfig = HttpsUntils.dwsClientConfig; + + private static final UJESClient client = new UJESClientImpl(clientConfig); + + @Scheduled(cron = "${linkis.monitor.user.cron}") + public void job() { + Optional.ofNullable(MonitorConfig.USER_MODE_ENGINE.getValue()).ifPresent(configStr -> { + ArrayList> userModeStr = + BDPJettyServerHelper.gson().fromJson(configStr, ArrayList.class); + userModeStr.forEach(engine -> { + // 3. build job and execute + JobExecuteResult jobExecuteResult = toSubmit(engine); + logger.info( + "start run engineType: {},job id : {}", + engine.get("engineType"), + jobExecuteResult.taskID()); + HashMap parms = new HashMap<>(); + parms.put("$engineType", engine.get("engineType")); + parms.put("$url", MonitorConfig.GATEWAY_URL.getValue()); + parms.put("$jobId", jobExecuteResult.taskID()); + Utils.sleepQuietly(MonitorConfig.USER_MODE_TIMEOUT.getValue() * 1000); + JobInfoResult jobInfo = client.getJobInfo(jobExecuteResult); + if (jobInfo.isCompleted()) { + if (jobInfo.getJobStatus().equals("Failed")) { + logger.info("run fail engineType: {},job id : {}", engine.get("engineType"), jobExecuteResult.taskID()); + RequestPersistTask requestPersistTask = jobInfo.getRequestPersistTask(); + parms.put("$errorCode", String.valueOf(requestPersistTask.getErrCode())); + parms.put("$errorMsg", requestPersistTask.getErrDesc()); + Map failedAlerts = MonitorAlertUtils.getAlerts(Constants.USER_RESOURCE_MONITOR(), parms); + PooledImsAlertUtils.addAlert(failedAlerts.get("12012")); + } + } else { + logger.info("run timeout engineType: {},job id : {}", engine.get("engineType"), jobExecuteResult.taskID()); + Map alerts = MonitorAlertUtils.getAlerts(Constants.USER_RESOURCE_MONITOR(), parms); + PooledImsAlertUtils.addAlert(alerts.get("12011")); + } + } + ); + }); + } + + private static JobExecuteResult toSubmit(LinkedTreeMap engine) { + // 1. build params + // set label map :EngineTypeLabel/UserCreatorLabel/EngineRunTypeLabel/Tenant + Map labels = new HashMap(); + labels.put(LabelKeyConstant.ENGINE_TYPE_KEY, engine.get("engineType")); // required engineType Label + labels.put(LabelKeyConstant.USER_CREATOR_TYPE_KEY, engine.get("executeUser") + "-IDE");// required execute user and creator eg:hadoop-IDE + labels.put(LabelKeyConstant.CODE_TYPE_KEY, engine.get("runType")); // required codeType + Map startupMap = new HashMap(16); + // setting linkis params + //startupMap.put("wds.linkis.rm.yarnqueue", "dws"); + // 2. build jobSubmitAction + JobSubmitAction jobSubmitAction = JobSubmitAction.builder() + .addExecuteCode(engine.get("code")) + .setStartupParams(startupMap) + .setUser(engine.get("executeUser")) //submit user + .addExecuteUser(engine.get("executeUser")) // execute user + .setLabels(labels) + .build(); + // 3. to execute + return client.submit(jobSubmitAction); + } + + @Scheduled(cron = "${linkis.monitor.user.db.cron:0 0/10 * * * ?}") + public void dbJob() { + Map properties= new HashMap<>(); + properties.put("readTimeout",MonitorConfig.USER_MODE_INTERFACE_TIMEOUT.getValue()); + DWSClientConfig clientConfig = HttpsUntils.createClientConfig(null, properties); + UJESClientImpl ujesClient = new UJESClientImpl(clientConfig); + GetTableStatisticInfoAction builder = GetTableStatisticInfoAction + .builder() + .setUser("hadoop") + .setDatabase("default") + .setTable("dual") + .builder(); + HashMap parms = new HashMap<>(); + try { + GetTableStatisticInfoResult tableStatisticInfo = ujesClient.getTableStatisticInfo(builder); + if (tableStatisticInfo.getStatus() != 0) { + logger.info("元数据查询服务用户态，执行失败，异常信息："+tableStatisticInfo.getMessage()); +// parms.put("$msg", tableStatisticInfo.getMessage()); +// Map failedAlerts = MonitorAlertUtils.getAlerts(Constants.USER_RESOURCE_MONITOR(), parms); +// PooledImsAlertUtils.addAlert(failedAlerts.get("12017")); + } + } catch (Exception e) { + if(e instanceof SocketTimeoutException){ + Integer timeoutValue = MonitorConfig.USER_MODE_INTERFACE_TIMEOUT.getValue(); + long timeout = TimeUnit.MILLISECONDS.toSeconds(timeoutValue); + logger.info("元数据查询服务用户态，执行超时："+timeout+"秒"); +// parms.put("$timeout", String.valueOf(timeout)); +// Map failedAlerts = MonitorAlertUtils.getAlerts(Constants.USER_RESOURCE_MONITOR(), parms); +// PooledImsAlertUtils.addAlert(failedAlerts.get("12018")); + } else { + logger.error("元数据查询服务用户态，执行异常："+ e); +// parms.put("$msg", e.getMessage()); +// Map failedAlerts = MonitorAlertUtils.getAlerts(Constants.USER_RESOURCE_MONITOR(), parms); +// PooledImsAlertUtils.addAlert(failedAlerts.get("12017")); + } + } + } +} diff --git a/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/monitor/scheduled/ValidatorClear.java b/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/monitor/scheduled/ValidatorClear.java new file mode 100644 index 0000000000..440b7a6bc4 --- /dev/null +++ b/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/monitor/scheduled/ValidatorClear.java @@ -0,0 +1,51 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.monitor.scan.app.monitor.scheduled; + +import org.apache.linkis.monitor.scan.app.monitor.config.MonitorConfig; +import org.apache.linkis.monitor.scan.app.monitor.until.ThreadUtils; +import org.apache.linkis.monitor.scan.utils.log.LogUtils; +import org.slf4j.Logger; +import org.springframework.context.annotation.PropertySource; +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.stereotype.Component; + +import java.util.ArrayList; +import java.util.List; +/*** + * Task: clean up linkis_et_validator_checkinfo data + */ + +@Component +@PropertySource(value = "classpath:linkis-et-monitor.properties", encoding = "UTF-8") +public class ValidatorClear { + + private static final Logger logger = LogUtils.stdOutLogger(); + + @Scheduled(cron = "${linkis.monitor.clear.validator.cron}") + public void ValidatorClear() { + logger.info("Start to clear_validator_record shell"); + List cmdlist = new ArrayList<>(); + cmdlist.add("sh"); + cmdlist.add(MonitorConfig.shellPath + "clear_validator_record.sh"); + logger.info("clear_validator_record shell command {}", cmdlist); + String exec = ThreadUtils.run(cmdlist, "clear_validator_record.sh"); + logger.info("shell log {}", exec); + logger.info("End to clear_validator_record shell "); + } +} diff --git a/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/monitor/until/CacheUtils.java b/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/monitor/until/CacheUtils.java new file mode 100644 index 0000000000..258d81183c --- /dev/null +++ b/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/monitor/until/CacheUtils.java @@ -0,0 +1,35 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.monitor.scan.app.monitor.until; + +import java.util.concurrent.TimeUnit; + +import com.google.common.cache.Cache; +import com.google.common.cache.CacheBuilder; + +public class CacheUtils { + + public static Cache cacheBuilder = + CacheBuilder.newBuilder() + .concurrencyLevel(5) + .expireAfterAccess(1, TimeUnit.DAYS) + .initialCapacity(20) + .maximumSize(1000) + .recordStats() + .build(); +} diff --git a/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/monitor/until/HttpsUntils.java b/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/monitor/until/HttpsUntils.java new file mode 100644 index 0000000000..1208a6ef64 --- /dev/null +++ b/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/monitor/until/HttpsUntils.java @@ -0,0 +1,183 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.monitor.scan.app.monitor.until; + +import org.apache.linkis.common.conf.Configuration; +import org.apache.linkis.common.utils.Utils; +import org.apache.linkis.httpclient.dws.authentication.TokenAuthenticationStrategy; +import org.apache.linkis.httpclient.dws.config.DWSClientConfig; +import org.apache.linkis.httpclient.dws.config.DWSClientConfigBuilder; +import org.apache.linkis.monitor.scan.app.monitor.config.MonitorConfig; +import org.apache.linkis.monitor.scan.app.monitor.entity.ChatbotEntity; +import org.apache.linkis.monitor.scan.app.monitor.entity.IndexEntity; +import org.apache.linkis.monitor.scan.client.MonitorHTTPClient; +import org.apache.linkis.monitor.scan.client.MonitorHTTPClientClientImpl; +import org.apache.linkis.monitor.scan.constants.Constants; +import org.apache.linkis.monitor.scan.request.EmsListAction; +import org.apache.linkis.monitor.scan.request.EntranceTaskAction; +import org.apache.linkis.monitor.scan.response.EntranceTaskResult; +import org.apache.linkis.server.BDPJettyServerHelper; + +import org.apache.commons.collections.MapUtils; +import org.apache.commons.lang3.StringUtils; +import org.apache.http.client.config.RequestConfig; +import org.apache.http.client.methods.CloseableHttpResponse; +import org.apache.http.client.methods.HttpPost; +import org.apache.http.entity.ContentType; +import org.apache.http.entity.StringEntity; +import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.impl.client.HttpClients; +import org.apache.http.util.EntityUtils; + +import org.apache.linkis.ujes.client.response.EmsListResult; +import org.springframework.util.Assert; + +import java.io.IOException; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.TimeUnit; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class HttpsUntils { + private static final Logger logger = LoggerFactory.getLogger(HttpsUntils.class); + + public static DWSClientConfig dwsClientConfig = createClientConfig(null, null); + // IOUtils.closeQuietly(client); + public static MonitorHTTPClient client = new MonitorHTTPClientClientImpl(dwsClientConfig); + public static final String localHost = Utils.getLocalHostname(); + + public static Map sendHttp(String url, Map properties) + throws IOException { + if (null == dwsClientConfig) { + dwsClientConfig = createClientConfig(url, properties); + } + if (null == client) { + client = new MonitorHTTPClientClientImpl(dwsClientConfig); + } + EmsListAction build = EmsListAction.newBuilder().setUser("hadoop").build(); + EmsListResult result = client.list(build); + return result.getResultMap(); + } + + public static DWSClientConfig createClientConfig(String url, Map properties) { + String realUrl = ""; + if (StringUtils.isBlank(url)) { + realUrl = Configuration.getGateWayURL(); + } else { + realUrl = url; + } + Map parms = new HashMap<>(); + if (MapUtils.isNotEmpty(properties)) { + parms = properties; + } + int maxConnection = + (int) + parms.getOrDefault( + Constants.CONNECTION_MAX_SIZE_SHORT_NAME(), + Constants.CONNECTION_MAX_SIZE().getValue()); + int connectTimeout = + (int) + parms.getOrDefault( + Constants.CONNECTION_TIMEOUT_SHORT_NAME(), + Constants.CONNECTION_TIMEOUT().getValue()); + int readTimeout = + (int) + parms.getOrDefault( + Constants.CONNECTION_READ_TIMEOUT_SHORT_NAME(), + Constants.CONNECTION_READ_TIMEOUT().getValue()); + String tokenKey = + (String) + parms.getOrDefault( + Constants.AUTH_TOKEN_KEY_SHORT_NAME(), Constants.AUTH_TOKEN_KEY().getValue()); + String tokenValue = + (String) + parms.getOrDefault( + Constants.AUTH_TOKEN_VALUE_SHORT_NAME(), Constants.AUTH_TOKEN_VALUE().getValue()); + + DWSClientConfig clientConfig = + ((DWSClientConfigBuilder) + (DWSClientConfigBuilder.newBuilder() + .addServerUrl(realUrl) + .connectionTimeout(connectTimeout) + .discoveryEnabled(false) + .discoveryFrequency(1, TimeUnit.MINUTES) + .loadbalancerEnabled(false) + .maxConnectionSize(maxConnection) + .retryEnabled(false) + .readTimeout(readTimeout) + .setAuthenticationStrategy(new TokenAuthenticationStrategy()) + .setAuthTokenKey(tokenKey) + .setAuthTokenValue(tokenValue))) + .setDWSVersion("v1") + .build(); + + return clientConfig; + } + + public static Map getEntranceTask(String url, String user,String Instance) throws IOException { + if (null == dwsClientConfig) { + dwsClientConfig = createClientConfig(null, null); + } + if (null == client) { + client = new MonitorHTTPClientClientImpl(dwsClientConfig); + } + EntranceTaskAction build = EntranceTaskAction.newBuilder().setUser(user).setInstance(Instance).build(); + EntranceTaskResult result = client.entranList(build); + return result.getResultMap(); + } + + public static void sendIndex(List list) throws IOException { + Map parm = new HashMap<>(); + parm.put("userAuthKey", MonitorConfig.ECM_TASK_USER_AUTHKEY.getValue()); + parm.put("metricDataList", list); + String json = BDPJettyServerHelper.gson().toJson(parm); + + RequestConfig requestConfig = RequestConfig.DEFAULT; + StringEntity entity = + new StringEntity( + json, ContentType.create(ContentType.APPLICATION_JSON.getMimeType(), "UTF-8")); + entity.setContentEncoding("UTF-8"); + + HttpPost httpPost = new HttpPost(MonitorConfig.ECM_TASK_IMURL.getValue()); + httpPost.setConfig(requestConfig); + httpPost.setEntity(entity); + + CloseableHttpClient httpClient = HttpClients.createDefault(); + CloseableHttpResponse execute = httpClient.execute(httpPost); + String responseStr = EntityUtils.toString(execute.getEntity(), "UTF-8"); + Map map = BDPJettyServerHelper.gson().fromJson(responseStr, Map.class); + logger.info("send index response :{}", map); + Assert.isTrue(!"0".equals(map.get("resultCode")), map.get("resultMsg")); + } + + public static void sendChatbot(ChatbotEntity chatbotEntity) throws IOException { + String json = BDPJettyServerHelper.gson().toJson(chatbotEntity); + StringEntity entity = new StringEntity(json, ContentType.create(ContentType.APPLICATION_JSON.getMimeType(), "UTF-8")); + entity.setContentEncoding("UTF-8"); + HttpPost httpPost = new HttpPost(MonitorConfig.CHATBOT_URL.getValue()); + httpPost.setConfig(RequestConfig.DEFAULT); + httpPost.setEntity(entity); + CloseableHttpResponse execute = HttpClients.createDefault().execute(httpPost); + String responseStr = EntityUtils.toString(execute.getEntity(), "UTF-8"); + Map map = BDPJettyServerHelper.gson().fromJson(responseStr, Map.class); + } + +} diff --git a/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/monitor/until/ThreadUtils.java b/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/monitor/until/ThreadUtils.java new file mode 100644 index 0000000000..5a099e7b65 --- /dev/null +++ b/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/monitor/until/ThreadUtils.java @@ -0,0 +1,68 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.monitor.scan.app.monitor.until; + +import org.apache.linkis.common.utils.Utils; +import org.apache.linkis.monitor.scan.app.monitor.config.MonitorConfig; +import org.apache.linkis.monitor.scan.constants.Constants; +import org.apache.linkis.monitor.scan.utils.alert.AlertDesc; +import org.apache.linkis.monitor.scan.utils.alert.ims.MonitorAlertUtils; +import org.apache.linkis.monitor.scan.utils.alert.ims.PooledImsAlertUtils; +import org.apache.linkis.monitor.scan.utils.log.LogUtils; + +import org.springframework.context.ApplicationContext; +import org.springframework.context.event.ApplicationContextEvent; + +import java.util.*; +import java.util.concurrent.*; + +import scala.concurrent.ExecutionContextExecutorService; + +import org.slf4j.Logger; + +public class ThreadUtils extends ApplicationContextEvent { + + private static final Logger logger = LogUtils.stdOutLogger(); + + public static ExecutionContextExecutorService executors = + Utils.newCachedExecutionContext(5, "alert-pool-thread-", false); + + public ThreadUtils(ApplicationContext source) { + super(source); + } + + public static String run(List cmdList, String shellName) { + FutureTask future = new FutureTask(() -> Utils.exec(cmdList.toArray(new String[2]), -1)); + executors.submit(future); + String msg = ""; + try { + msg = future.get(MonitorConfig.SHELL_TIMEOUT.getValue(), TimeUnit.MINUTES).toString(); + } catch (TimeoutException e) { + // 增加告警提示 + logger.info("超时告警 {}", shellName); + HashMap parms = new HashMap<>(); + parms.put("$shellName", shellName); + Map ecmResourceAlerts = + MonitorAlertUtils.getAlerts(Constants.THREAD_TIME_OUT_IM(), parms); + PooledImsAlertUtils.addAlert(ecmResourceAlerts.get("12014")); + } catch (ExecutionException | InterruptedException e) { + logger.error("Thread error msg {}", e.getMessage()); + } + return msg; + } +} diff --git a/linkis-extensions/linkis-et-monitor/src/main/resources/mapper/common/InsLabelRelationMapper.xml b/linkis-extensions/linkis-et-monitor/src/main/resources/mapper/common/InsLabelRelationMapper.xml new file mode 100644 index 0000000000..b9c1d19612 --- /dev/null +++ b/linkis-extensions/linkis-et-monitor/src/main/resources/mapper/common/InsLabelRelationMapper.xml @@ -0,0 +1,67 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + l.`id`, l.`label_key`, l.`label_value`, l.`label_feature`, + l.`label_value_size`, l.`update_time`, l.`create_time` + + + + s.`id`, s.`instance`, s.`name`, s.`update_time`, s.`create_time` + + + + + + + + + \ No newline at end of file diff --git a/linkis-extensions/linkis-et-monitor/src/main/resources/mapper/common/InstanceInfoMapper.xml b/linkis-extensions/linkis-et-monitor/src/main/resources/mapper/common/InstanceInfoMapper.xml new file mode 100644 index 0000000000..c0ab7f2b22 --- /dev/null +++ b/linkis-extensions/linkis-et-monitor/src/main/resources/mapper/common/InstanceInfoMapper.xml @@ -0,0 +1,53 @@ + + + + + + + + + + + + + + + + `id`, `instance`, `name`, `update_time`, + `create_time` + + + + DELETE FROM linkis_ps_instance_info WHERE instance = #{instance} + + + + + + + \ No newline at end of file diff --git a/linkis-extensions/linkis-et-monitor/src/main/resources/mapper/common/InstanceLabelMapper.xml b/linkis-extensions/linkis-et-monitor/src/main/resources/mapper/common/InstanceLabelMapper.xml new file mode 100644 index 0000000000..d92d999d47 --- /dev/null +++ b/linkis-extensions/linkis-et-monitor/src/main/resources/mapper/common/InstanceLabelMapper.xml @@ -0,0 +1,57 @@ + + + + + + + + + + + + + + + + + + `id`, `label_key`, `label_value`, `label_feature`, + `label_value_size`, `update_time`, `create_time` + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/linkis-extensions/linkis-et-monitor/src/main/resources/mapper/common/JobHistoryMapper.xml b/linkis-extensions/linkis-et-monitor/src/main/resources/mapper/common/JobHistoryMapper.xml new file mode 100644 index 0000000000..70d1f30158 --- /dev/null +++ b/linkis-extensions/linkis-et-monitor/src/main/resources/mapper/common/JobHistoryMapper.xml @@ -0,0 +1,172 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + `id`,`job_req_id`,`submit_user`,`execute_user`,`labels`,`params`,`status`,`error_code`,`created_time`, + `updated_time`,`instances`,`observe_info` + + + + + + + + UPDATE linkis_ps_job_history_group_history + + status = #{targetStatus} + + + + #{element} + + + + + + UPDATE linkis_ps_job_history_group_history + + status = #{targetStatus}, error_code=21304, error_desc='Automatically killed because entrance is dead' + + + created_time >= #{startDate} + AND instances = #{instanceName} + AND + + #{element} + + + LIMIT 5000 + + + + + + + + + + diff --git a/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/app/LinkisJobHistoryScanSpringConfiguration.scala b/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/app/LinkisJobHistoryScanSpringConfiguration.scala new file mode 100644 index 0000000000..7ee159d936 --- /dev/null +++ b/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/app/LinkisJobHistoryScanSpringConfiguration.scala @@ -0,0 +1,49 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.monitor.scan.app + +import org.apache.linkis.monitor.scan.app.factory.MapperFactory +import org.apache.linkis.monitor.scan.app.instance.dao.InstanceInfoDao +import org.apache.linkis.monitor.scan.app.jobhistory.dao.JobHistoryMapper + +import org.springframework.beans.factory.annotation.Autowired +import org.springframework.context.annotation.{ComponentScan, Configuration} + +import javax.annotation.PostConstruct + +/** + * Created by shangda on 2021/11/19. + */ + +@Configuration +@ComponentScan(Array("org.apache.linkis.monitor.scan", "org.apache.linkis.mybatis")) +class LinkisJobHistoryScanSpringConfiguration { + + @Autowired + private var jobHistoryMapper: JobHistoryMapper = _ + + @Autowired + private var instanceInfoMapper: InstanceInfoDao = _ + + @PostConstruct + def init(): Unit = { + MapperFactory.setJobHistoryMapper(jobHistoryMapper) + MapperFactory.setInstanceInfoMapper(instanceInfoMapper) + } + +} diff --git a/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/app/factory/MapperFactory.scala b/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/app/factory/MapperFactory.scala new file mode 100644 index 0000000000..337592bf72 --- /dev/null +++ b/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/app/factory/MapperFactory.scala @@ -0,0 +1,64 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.monitor.scan.app.factory + +import org.apache.linkis.monitor.scan.app.instance.dao.{ + InsLabelRelationDao, + InstanceInfoDao, + InstanceLabelDao +} +import org.apache.linkis.monitor.scan.app.instance.dao.InstanceInfoDao +import org.apache.linkis.monitor.scan.app.jobhistory.dao.JobHistoryMapper + + +object MapperFactory { + // val bmlVersionCleanScanOper = new BmlVersionCleanScanOper + + private var jobHistoryMapper: JobHistoryMapper = _ + + private var instanceInfoMapper: InstanceInfoDao = _ + + private var instanceLabelMapper: InstanceLabelDao = _ + + private var instanceLabelRelationMapper: InsLabelRelationDao = _ + + def getJobHistoryMapper() = jobHistoryMapper + + def setJobHistoryMapper(jobHistoryMapper: JobHistoryMapper) = { + MapperFactory.jobHistoryMapper = jobHistoryMapper + } + + def getInstanceInfoMapper() = instanceInfoMapper + + def setInstanceInfoMapper(instanceInfoMapper: InstanceInfoDao) = { + MapperFactory.instanceInfoMapper = instanceInfoMapper + } + + def getInstanceLabelMapper() = instanceLabelMapper + + def setInstanceLabelMapper(instanceLabelMapper: InstanceLabelDao) = { + MapperFactory.instanceLabelMapper = instanceLabelMapper + } + + def getInsLabelRelationMapper() = instanceLabelRelationMapper + + def setInsLabelRelationMapper(instanceLabelRelationMapper: InsLabelRelationDao) = { + MapperFactory.instanceLabelRelationMapper = instanceLabelRelationMapper + } + +} diff --git a/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/app/jobhistory/JobHistoryDataFetcher.scala b/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/app/jobhistory/JobHistoryDataFetcher.scala new file mode 100644 index 0000000000..b8eff63ec8 --- /dev/null +++ b/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/app/jobhistory/JobHistoryDataFetcher.scala @@ -0,0 +1,111 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.monitor.scan.app.jobhistory + +import java.util +import java.util.Date + +import org.apache.commons.lang3.StringUtils +import org.apache.linkis.common.utils.{Logging, Utils} +import org.apache.linkis.monitor.scan.app.jobhistory.dao.JobHistoryMapper +import org.apache.linkis.monitor.scan.app.jobhistory.exception.AnomalyScannerException +import org.apache.linkis.monitor.scan.constants.Constants +import org.apache.linkis.monitor.scan.core.pac.AbstractDataFetcher + + +class JobHistoryDataFetcher(args: Array[Any], mapper: JobHistoryMapper) + extends AbstractDataFetcher + with Logging { + + /** + * retrieve JobHistory Data starts from startTimeMs and ends at startTimeMs + intervalsMs + * + * @return + */ + /** + * get arguments for querying data + * + * @return + */ + override def getArgs(): Array[Any] = args + + /** + * 1. get Data given some arguments + */ + override def getData(): util.List[scala.Any] = { + if (!args.isInstanceOf[Array[String]]) { + throw new AnomalyScannerException( + 21304, + "Wrong input for JobHistoryDataFetcher. DataType: " + args.getClass.getCanonicalName + ) + } + if (args != null && args.length == 2) { + val start = Utils.tryCatch(args(0).asInstanceOf[String].toLong) { t => + { + logger.error("Failed to get data from DB: Illegal arguments.", t) + throw t + } + } + val end = Utils.tryCatch(args(1).asInstanceOf[String].toLong) { t => + { + logger.error("Failed to get data from DB: Illegal arguments.", t) + throw t + } + } + mapper + .search(null, null, null, new Date(start), new Date(end), null) + .asInstanceOf[util.List[scala.Any]] + } else if (args != null && args.length == 4) { + val start = Utils.tryCatch(args(0).asInstanceOf[String].toLong) { t => + { + logger.error("Failed to get data from DB: Illegal arguments.", t) + throw t + } + } + val end = Utils.tryCatch(args(1).asInstanceOf[String].toLong) { t => + { + logger.error("Failed to get data from DB: Illegal arguments.", t) + throw t + } + } + val id = Utils.tryCatch(args(2).asInstanceOf[String].toLong) { t => + { + logger.error("Failed to get data from DB: Illegal arguments.", t) + throw t + } + } + if (StringUtils.isNotBlank(args(3).asInstanceOf[String]) && args(3).asInstanceOf[String].equals("updated_time")) { + val list = new util.ArrayList[String]() + Constants.DIRTY_DATA_FINISHED_JOB_STATUS_ARRAY.foreach(list.add(_)) + mapper + .searchByCacheAndUpdateTime(id, null, list, new Date(start), new Date(end), null) + .asInstanceOf[util.List[scala.Any]] + } else { + mapper + .searchByCache(id, null, null, new Date(start), new Date(end), null) + .asInstanceOf[util.List[scala.Any]] + } + } else { + throw new AnomalyScannerException( + 21304, + "Wrong input for JobHistoryDataFetcher. Data: " + args + ) + } + } + +} diff --git a/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/app/jobhistory/errorcode/JobHistoryErrCodeHitEvent.scala b/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/app/jobhistory/errorcode/JobHistoryErrCodeHitEvent.scala new file mode 100644 index 0000000000..f899952c71 --- /dev/null +++ b/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/app/jobhistory/errorcode/JobHistoryErrCodeHitEvent.scala @@ -0,0 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.monitor.scan.app.jobhistory.errorcode + +import org.apache.linkis.monitor.scan.core.ob.SingleObserverEvent + + +class JobHistoryErrCodeHitEvent extends SingleObserverEvent diff --git a/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/app/jobhistory/errorcode/JobHistoryErrCodeRule.scala b/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/app/jobhistory/errorcode/JobHistoryErrCodeRule.scala new file mode 100644 index 0000000000..c613b3d306 --- /dev/null +++ b/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/app/jobhistory/errorcode/JobHistoryErrCodeRule.scala @@ -0,0 +1,80 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.monitor.scan.app.jobhistory.errorcode + +import java.util + +import org.apache.linkis.common.utils.Logging +import org.apache.linkis.monitor.scan.app.jobhistory.entity.JobHistory +import org.apache.linkis.monitor.scan.app.monitor.until.CacheUtils +import org.apache.linkis.monitor.scan.core.ob.Observer +import org.apache.linkis.monitor.scan.core.pac.{AbstractScanRule, ScannedData} +import scala.collection.JavaConverters._ + + +/** + * 针对执行任务返回的错误码进行监控，执行脚本任务时，会记录执行的错误码在数据库中， + * 服务会根据数据库中记录的错误码，来进行告警，如果错误码中包含（11001，11002）即可触发告警 + */ +class JobHistoryErrCodeRule(errorCodes: util.Set[String], hitObserver: Observer) + extends AbstractScanRule(event = new JobHistoryErrCodeHitEvent, observer = hitObserver) + with Logging { + private val scanRuleList = CacheUtils.cacheBuilder + + /** + * if data match the pattern, return true and trigger observer should call isMatched() + * + * @param data + * @return + */ + override def triggerIfMatched(data: util.List[ScannedData]): Boolean = { + + if (!getHitEvent().isRegistered || null == data) { + logger.error("ScanRule is not bind with an observer. Will not be triggered") + return false + } + + val alertData: util.List[JobHistory] = new util.ArrayList[JobHistory]() + for (sd <- data.asScala) { + if (sd != null && sd.getData() != null) { + for (d <- sd.getData().asScala) { + d match { + case history: JobHistory => + if (errorCodes.contains(String.valueOf(history.getErrorCode))) { + alertData.add(history) + } + scanRuleList.put("jobHistoryId", history.getId) + case _ => + logger.warn("Ignored wrong input data Type : " + d + ", " + d.getClass.getCanonicalName) + } + } + } else { + logger.warn("Ignored null scanned data") + } + + } + logger.info("hit " + alertData.size() + " data in one iteration") + if (alertData.size() > 0) { + getHitEvent().notifyObserver(getHitEvent(), alertData) + true + } else { + false + } + } + +} diff --git a/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/app/jobhistory/errorcode/JobHistoryErrorCodeAlertSender.scala b/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/app/jobhistory/errorcode/JobHistoryErrorCodeAlertSender.scala new file mode 100644 index 0000000000..a037e0f017 --- /dev/null +++ b/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/app/jobhistory/errorcode/JobHistoryErrorCodeAlertSender.scala @@ -0,0 +1,97 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.monitor.scan.app.jobhistory.errorcode + +import java.util + +import org.apache.linkis.common.utils.Logging +import org.apache.linkis.monitor.scan.app.jobhistory.entity.JobHistory +import org.apache.linkis.monitor.scan.app.jobhistory.exception.AnomalyScannerException +import org.apache.linkis.monitor.scan.core.ob.{Event, Observer} +import org.apache.linkis.monitor.scan.utils.alert.AlertDesc +import org.apache.linkis.monitor.scan.utils.alert.ims.{ImsAlertDesc, PooledImsAlertUtils} +import scala.collection.JavaConverters._ + + + +class JobHistoryErrorCodeAlertSender(alerts: util.Map[String, AlertDesc]) + extends Observer + with Logging { + + override def update(e: Event, jobHistoryList: scala.Any): Unit = { + if (!e.isInstanceOf[JobHistoryErrCodeHitEvent]) { + throw new AnomalyScannerException( + 21304, + "Wrong event that triggers JobHistoryErrorCodeAlertSender. Input DataType: " + e.getClass.getCanonicalName + ) + } + if (null == jobHistoryList || !jobHistoryList.isInstanceOf[util.List[_]]) { + throw new AnomalyScannerException( + 21304, + "Wrong input for JobHistoryErrorCodeAlertSender. Input DataType: " + jobHistoryList.getClass.getCanonicalName + ) + } + val toSend = new util.HashMap[String, ImsAlertDesc] + for (a <- jobHistoryList.asInstanceOf[util.List[_]].asScala) { + if (a == null) { + logger.warn("Ignore null input data") + } else if (!a.isInstanceOf[JobHistory]) { + logger.warn("Ignore wrong input data Type : " + a.getClass.getCanonicalName) + } else { + val jobHistory = a.asInstanceOf[JobHistory] + val errorCode = String.valueOf(jobHistory.getErrorCode) + if (alerts.containsKey(errorCode) && alerts.get(errorCode).isInstanceOf[ImsAlertDesc]) { + val alert = if (!toSend.containsKey(errorCode)) { + alerts.get(errorCode).asInstanceOf[ImsAlertDesc] + } else { + toSend.get(errorCode) + } + + var newInfo = if (!toSend.containsKey(errorCode)) { + alert.alertInfo + "\n" + + "[error_code] " + jobHistory.getErrorCode + ", " + jobHistory.getErrorDesc + "\n" + } else { + alert.alertInfo + } + newInfo = newInfo + + "[job-info] " + + "submit-user: " + jobHistory.getSubmitUser + ", " + + "execute-user: " + jobHistory.getExecuteUser + ", " + + "engine_type: " + jobHistory.getEngineType + ", " + + "create_time: " + jobHistory.getCreatedTime + ", " + + "instance: " + jobHistory.getInstances + ". \n" + val newNumHit = alert.numHit + 1 + toSend.put(errorCode, alert.copy(alertInfo = newInfo, numHit = newNumHit)) + } else if (!alerts.containsKey(errorCode)) { + logger.warn("Ignored unregistered error code: " + errorCode) + } else if (!alerts.get(errorCode).isInstanceOf[ImsAlertDesc]) { + logger.warn( + "Ignored invalid alertDesc. DataType: " + alerts + .get(errorCode) + .getClass + .getCanonicalName + ) + } + } + } + for ((_, alert) <- toSend.asScala) { + PooledImsAlertUtils.addAlert(alert) + } + } + +} diff --git a/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/app/jobhistory/jobtime/JobTimeExceedAlertSender.scala b/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/app/jobhistory/jobtime/JobTimeExceedAlertSender.scala new file mode 100644 index 0000000000..aa564ab335 --- /dev/null +++ b/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/app/jobhistory/jobtime/JobTimeExceedAlertSender.scala @@ -0,0 +1,112 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.monitor.scan.app.jobhistory.jobtime + +import java.text.MessageFormat +import java.util + +import org.apache.linkis.common.utils.{Logging, Utils} +import org.apache.linkis.monitor.scan.app.jobhistory.entity.JobHistory +import org.apache.linkis.monitor.scan.app.jobhistory.exception.AnomalyScannerException +import org.apache.linkis.monitor.scan.app.monitor.config.MonitorConfig +import org.apache.linkis.monitor.scan.core.ob.{Event, Observer} +import org.apache.linkis.monitor.scan.utils.alert.AlertDesc +import org.apache.linkis.monitor.scan.utils.alert.ims.{ImsAlertDesc, PooledImsAlertUtils} + +import scala.collection.JavaConverters._ +import scala.collection.mutable.ArrayBuffer + + +class JobTimeExceedAlertSender(alerts: util.Map[String, AlertDesc]) + extends Observer + with Logging { + + private val orderedThresholds: Array[Long] = { + val ret = new ArrayBuffer[Long]() + if (alerts != null) { + for (k <- alerts.keySet().asScala) { + Utils.tryCatch(ret.append(k.toLong)) { t => + logger.warn("Ignored illegal threshold: " + k, t) + false + } + } + } + ret.toArray + } + + override def update(e: Event, jobHistoryList: scala.Any): Unit = { + if (!e.isInstanceOf[JobTimeExceedHitEvent]) { + throw new AnomalyScannerException( + 21304, + "Wrong event that triggers JobTimeExceedAlertSender. Input DataType: " + e.getClass.getCanonicalName + ) + } + if (null == jobHistoryList || !jobHistoryList.isInstanceOf[util.List[_]]) { + throw new AnomalyScannerException( + 21304, + "Wrong input for JobTimeExceedAlertSender. Input DataType: " + jobHistoryList.getClass.getCanonicalName + ) + } + if (orderedThresholds.length == 0) { + logger.warn("Found none legal threshold, will not send any alert: " + this) + return + } + val toSend = new util.HashMap[String, ImsAlertDesc] + for (a <- jobHistoryList.asInstanceOf[util.List[_]].asScala) { + if (a == null) { + logger.warn("Ignore null input data") + } else if (!a.isInstanceOf[JobHistory]) { + logger.warn("Ignore wrong input data Type : " + a.getClass.getCanonicalName) + } else { + val jobHistory = a.asInstanceOf[JobHistory] + val elapse = System.currentTimeMillis() - jobHistory.getCreatedTime.getTime + var ts = 0L + for (t <- orderedThresholds) { // search max threshold that is smaller than elapse + if (elapse >= t) { + ts = t + } else { + + } + } + val name = ts.toString + val alert = if (!toSend.containsKey(name)) { + alerts + .get(name) + .asInstanceOf[ + ImsAlertDesc + ] + } else { + toSend.get(name) + } + + val newInfo = MessageFormat.format("[Linkis任务信息]您好，您在Linkis/DSS提交的任务(任务ID:{0})，已经运行超过{1}h，" + + "请关注是否任务正常，如果不正常您可以到Linkis/DSS管理台进行任务的kill，集群信息为BDAP({2})。详细解决方案见Q47：{3} " + , jobHistory.getId, (elapse / 1000 / 60 / 60).toString, jobHistory.getInstances, MonitorConfig.SOLUTION_URL.getValue) + + val newNumHit = alert.numHit + 1 + val receiver = new util.HashSet[String]() + receiver.add(jobHistory.getSubmitUser) + receiver.add(jobHistory.getExecuteUser) + receiver.addAll(alert.alertReceivers) + val ImsAlertDesc = alert.copy(alertInfo = newInfo, alertReceivers = receiver, numHit = newNumHit) + PooledImsAlertUtils.addAlert(ImsAlertDesc) + + } + } + } +} diff --git a/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/app/jobhistory/jobtime/JobTimeExceedHitEvent.scala b/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/app/jobhistory/jobtime/JobTimeExceedHitEvent.scala new file mode 100644 index 0000000000..b7b883e09b --- /dev/null +++ b/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/app/jobhistory/jobtime/JobTimeExceedHitEvent.scala @@ -0,0 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.monitor.scan.app.jobhistory.jobtime + +import org.apache.linkis.monitor.scan.core.ob.SingleObserverEvent + + +class JobTimeExceedHitEvent extends SingleObserverEvent diff --git a/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/app/jobhistory/jobtime/JobTimeExceedRule.scala b/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/app/jobhistory/jobtime/JobTimeExceedRule.scala new file mode 100644 index 0000000000..b91d605d4e --- /dev/null +++ b/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/app/jobhistory/jobtime/JobTimeExceedRule.scala @@ -0,0 +1,103 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.monitor.scan.app.jobhistory.jobtime + +import java.util +import java.util.Locale + +import org.apache.linkis.common.utils.Logging +import org.apache.linkis.monitor.scan.app.jobhistory.entity.JobHistory +import org.apache.linkis.monitor.scan.app.jobhistory.exception.AnomalyScannerException +import org.apache.linkis.monitor.scan.app.monitor.until.CacheUtils +import org.apache.linkis.monitor.scan.constants.Constants +import org.apache.linkis.monitor.scan.core.ob.Observer +import org.apache.linkis.monitor.scan.core.pac.{AbstractScanRule, ScannedData} + +import scala.collection.JavaConverters._ + +/** + * 针对执行任务状态进行监控，扫描12小时之外，24小时之内的的数据， + * 如果规则范围内，有数据状态是（Inited,WaitForRetry,Scheduled,Running）其中之一，则触发告警 + */ +class JobTimeExceedRule(thresholds: util.Set[String], hitObserver: Observer) + extends AbstractScanRule(event = new JobTimeExceedHitEvent, observer = hitObserver) + with Logging { + + private val threshold: Long = { + if (thresholds == null) { + throw new AnomalyScannerException(21304, "thresholds should not be null") + } + var t = Long.MaxValue + for (k <- thresholds.asScala) { + if (k != null) { + if (t > k.toLong) { + t = k.toLong + } + } else { + logger.warn("ignored null input") + } + } + t + } + + private val scanRuleList = CacheUtils.cacheBuilder + + /** + * if data match the pattern, return true and trigger observer should call isMatched() + * + * @param data + * @return + */ + override def triggerIfMatched(data: util.List[ScannedData]): Boolean = { + if (!getHitEvent.isRegistered || data == null) { + logger.error("ScanRule is not bind with an observer. Will not be triggered") + return false + } + val alertData: util.List[JobHistory] = new util.ArrayList[JobHistory]() + for (sd <- data.asScala) { + if (sd != null && sd.getData() != null) { + for (d <- sd.getData().asScala) { + if (d.isInstanceOf[JobHistory]) { + val jobHistory = d.asInstanceOf[JobHistory] + val status = jobHistory.getStatus.toUpperCase(Locale.getDefault) + if (Constants.DIRTY_DATA_UNFINISHED_JOB_STATUS.contains(status)) { + val elapse = System.currentTimeMillis() - jobHistory.getCreatedTime.getTime + if (elapse / 1000 >= threshold) { + alertData.add(d.asInstanceOf[JobHistory]) + } + } + scanRuleList.put("jobhistoryScan", jobHistory.getId) + } else { + logger.warn("Ignored wrong input data Type : " + d + ", " + d.getClass.getCanonicalName) + } + } + } else { + logger.warn("Ignored null scanned data") + } + + } + logger.info("hit " + alertData.size() + " data in one iteration") + if (alertData.size() > 0) { + getHitEvent.notifyObserver(getHitEvent, alertData) + true + } else { + false + } + } + +} diff --git a/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/app/jobhistory/labels/JobHistoryLabelsAlertSender.scala b/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/app/jobhistory/labels/JobHistoryLabelsAlertSender.scala new file mode 100644 index 0000000000..77bc29ffe2 --- /dev/null +++ b/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/app/jobhistory/labels/JobHistoryLabelsAlertSender.scala @@ -0,0 +1,69 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.monitor.scan.app.jobhistory.labels + +import java.util + +import org.apache.linkis.common.utils.Logging +import org.apache.linkis.monitor.scan.app.jobhistory.entity.JobHistory +import org.apache.linkis.monitor.scan.app.jobhistory.exception.AnomalyScannerException +import org.apache.linkis.monitor.scan.constants.Constants +import org.apache.linkis.monitor.scan.core.ob.{Event, Observer} +import org.apache.linkis.monitor.scan.utils.alert.AlertDesc +import org.apache.linkis.monitor.scan.utils.alert.ims.{PooledImsAlertUtils, UserLabelAlertUtils} +import org.apache.linkis.server.BDPJettyServerHelper + +import scala.collection.JavaConverters._ +import scala.collection.mutable.ArrayBuffer + +class JobHistoryLabelsAlertSender() extends Observer with Logging { + + override def update(e: Event, jobHistoryList: scala.Any): Unit = { + if (!e.isInstanceOf[JobHistoryLabelsHitEvent]) { + throw new AnomalyScannerException( + 21304, + "Wrong event that triggers JobHistoryLabelsAlertSender. Input DataType: " + e.getClass.getCanonicalName + ) + } + if (null == jobHistoryList || !jobHistoryList.isInstanceOf[util.List[_]]) { + throw new AnomalyScannerException( + 21304, + "Wrong input for JobHistoryLabelsAlertSender. Input DataType: " + jobHistoryList.getClass.getCanonicalName + ) + } + val toSend = new ArrayBuffer[String] + for (a <- jobHistoryList.asInstanceOf[util.List[_]].asScala) { + if (a == null) { + logger.warn("Ignore null input data") + } else if (!a.isInstanceOf[JobHistory]) { + logger.warn("Ignore wrong input data Type : " + a.getClass.getCanonicalName) + } else { + val jobHistory = a.asInstanceOf[JobHistory] + toSend.append(jobHistory.getLabels) + } + } + for (str <- toSend.distinct) { + val labelsMap: util.Map[String, String] = + BDPJettyServerHelper.gson.fromJson(str, classOf[java.util.Map[String, String]]) + val alerts: util.Map[String, AlertDesc] = + UserLabelAlertUtils.getAlerts(Constants.USER_LABEL_MONITOR, labelsMap.get("userCreator")) + PooledImsAlertUtils.addAlert(alerts.get("12010")); + } + } + +} diff --git a/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/app/jobhistory/labels/JobHistoryLabelsHitEvent.scala b/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/app/jobhistory/labels/JobHistoryLabelsHitEvent.scala new file mode 100644 index 0000000000..6f47136acd --- /dev/null +++ b/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/app/jobhistory/labels/JobHistoryLabelsHitEvent.scala @@ -0,0 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.monitor.scan.app.jobhistory.labels + +import org.apache.linkis.monitor.scan.core.ob.SingleObserverEvent + +class JobHistoryLabelsHitEvent extends SingleObserverEvent diff --git a/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/app/jobhistory/labels/JobHistoryLabelsRule.scala b/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/app/jobhistory/labels/JobHistoryLabelsRule.scala new file mode 100644 index 0000000000..6308ac28b0 --- /dev/null +++ b/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/app/jobhistory/labels/JobHistoryLabelsRule.scala @@ -0,0 +1,112 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.monitor.scan.app.jobhistory.labels + +import java.util + +import com.google.common.collect.HashBiMap +import org.apache.commons.lang3.StringUtils +import org.apache.linkis.common.utils.Logging +import org.apache.linkis.monitor.scan.app.jobhistory.entity.JobHistory +import org.apache.linkis.monitor.scan.app.monitor.until.CacheUtils +import org.apache.linkis.monitor.scan.constants.Constants +import org.apache.linkis.monitor.scan.core.ob.Observer +import org.apache.linkis.monitor.scan.core.pac.{AbstractScanRule, ScannedData} +import org.apache.linkis.server.BDPJettyServerHelper + +import scala.collection.JavaConverters._ + +/** + * 对前20分钟内的执行数据进行扫描，对数据的labels字段进行判断, + * 判断依据monitor配置（linkis.monitor.jobhistory.userLabel.tenant） + */ +class JobHistoryLabelsRule(hitObserver: Observer) + extends AbstractScanRule(event = new JobHistoryLabelsHitEvent, observer = hitObserver) + with Logging { + + private val scanRuleList = CacheUtils.cacheBuilder + + /** + * if data match the pattern, return true and trigger observer should call isMatched() + * + * @param data + * @return + */ + override def triggerIfMatched(data: util.List[ScannedData]): Boolean = { + if (!getHitEvent.isRegistered || null == data) { + logger.error("ScanRule is not bind with an observer. Will not be triggered") + return false + } + val alertData: util.List[JobHistory] = new util.ArrayList[JobHistory]() + for (sd <- data.asScala) { + if (sd != null && sd.getData() != null) { + for (d <- sd.getData().asScala) { + if (d.isInstanceOf[JobHistory]) { + logger.info(" start jobhistory user label rule data : {}", d) + val jobHistory = d.asInstanceOf[JobHistory] + val labels = jobHistory.getLabels + val labelsMap: util.Map[String, String] = + BDPJettyServerHelper.gson.fromJson(labels, classOf[java.util.Map[String, String]]) + val userCreator = labelsMap.get("userCreator"); + val tenant = labelsMap.get("tenant"); + if (StringUtils.isNotBlank(userCreator)) { + val configMap = BDPJettyServerHelper.gson.fromJson( + Constants.USER_LABEL_TENANT.getValue, + classOf[java.util.Map[String, String]] + ) + // 当任务的creator是qualitis（或dops）时，tenant不是qualitis发出告警 + val listIterator = configMap.keySet.iterator + while ({ + listIterator.hasNext + }) { + val next = listIterator.next + if (userCreator.contains(next)) { + val value = configMap.get(next) + if (!value.equals(tenant)) { + alertData.add(d.asInstanceOf[JobHistory]) + } + } + } + // 当任务代理tenant:Qualitis标签，但是creator不是qualitis标签也进行告警 + if (configMap.values().contains(tenant)) { + val bimap: HashBiMap[String, String] = HashBiMap.create(configMap) + val key = bimap.inverse().get(tenant) + if (!key.contains(userCreator)) { + alertData.add(d.asInstanceOf[JobHistory]) + } + } + } + scanRuleList.put("jobHistoryId", jobHistory.getId) + } else { + logger.warn("Ignored wrong input data Type : " + d + ", " + d.getClass.getCanonicalName) + } + } + } else { + logger.warn("Ignored null scanned data") + } + } + logger.info("hit " + alertData.size() + " data in one iteration") + if (alertData.size() > 0) { + getHitEvent.notifyObserver(getHitEvent, alertData) + true + } else { + false + } + } + +} diff --git a/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/app/jobhistory/runtime/CommonJobRunTimeRule.scala b/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/app/jobhistory/runtime/CommonJobRunTimeRule.scala new file mode 100644 index 0000000000..1912acf8b4 --- /dev/null +++ b/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/app/jobhistory/runtime/CommonJobRunTimeRule.scala @@ -0,0 +1,79 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.monitor.scan.app.jobhistory.runtime + +import org.apache.commons.lang3.StringUtils +import org.apache.linkis.common.utils.Logging +import org.apache.linkis.monitor.scan.app.jobhistory.entity.JobHistory +import org.apache.linkis.monitor.scan.constants.Constants +import org.apache.linkis.monitor.scan.core.ob.Observer +import org.apache.linkis.monitor.scan.core.pac.{AbstractScanRule, ScannedData} + +import java.util +import scala.collection.JavaConverters._ + +/** + * 对前20分钟内的执行数据进行扫描， + * 1.数据的ObserveInfo字段进行判断是否为空， + * 2.任务状态已经完成（Succeed,Failed,Cancelled,Timeout,ALL） + * 满足条件即可触发告警 + */ +class CommonJobRunTimeRule(hitObserver: Observer) + extends AbstractScanRule(event = new JobHistoryRunTimeHitEvent, observer = hitObserver) + with Logging { + + /** + * if data match the pattern, return true and trigger observer should call isMatched() + * + * @param data + * @return + */ + override def triggerIfMatched(data: util.List[ScannedData]): Boolean = { + if (!getHitEvent.isRegistered || null == data) { + logger.error("ScanRule is not bind with an observer. Will not be triggered") + return false + } + val alertData: util.List[JobHistory] = new util.ArrayList[JobHistory]() + for (sd <- data.asScala) { + if (sd != null && sd.getData() != null) { + for (d <- sd.getData().asScala) { + d match { + case jobHistory: JobHistory => + if (Constants.DIRTY_DATA_FINISHED_JOB_STATUS.contains(jobHistory.getStatus.toUpperCase()) + &&StringUtils.isNotBlank(jobHistory.getObserveInfo)) { + alertData.add(jobHistory) + } else { + logger.warn("jobHistory is not completely ， taskid :" + d) + } + case _ => + } + } + } else { + logger.warn("Ignored null scanned data") + } + } + logger.info("hit " + alertData.size() + " data in one iteration") + if (alertData.size() > 0) { + getHitEvent.notifyObserver(getHitEvent, alertData) + true + } else { + false + } + } + +} diff --git a/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/app/jobhistory/runtime/CommonRunTimeAlertSender.scala b/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/app/jobhistory/runtime/CommonRunTimeAlertSender.scala new file mode 100644 index 0000000000..6aca4c38ff --- /dev/null +++ b/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/app/jobhistory/runtime/CommonRunTimeAlertSender.scala @@ -0,0 +1,88 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.monitor.scan.app.jobhistory.runtime + +import java.net.InetAddress +import java.text.SimpleDateFormat +import java.util +import java.util.Date + +import org.apache.commons.collections.MapUtils +import org.apache.linkis.common.utils.Logging +import org.apache.linkis.monitor.scan.app.jobhistory.entity.JobHistory +import org.apache.linkis.monitor.scan.app.jobhistory.exception.AnomalyScannerException +import org.apache.linkis.monitor.scan.constants.Constants +import org.apache.linkis.monitor.scan.core.ob.{Event, Observer} +import org.apache.linkis.monitor.scan.utils.alert.ims.{MonitorAlertUtils, PooledImsAlertUtils} +import org.apache.linkis.server.BDPJettyServerHelper + +import scala.collection.JavaConverters._ + + +class CommonRunTimeAlertSender() + extends Observer + with Logging { + private val dateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss") + + override def update(e: Event, jobHistoryList: scala.Any): Unit = { + if (!e.isInstanceOf[JobHistoryRunTimeHitEvent]) { + throw new AnomalyScannerException( + 21304, + "Wrong event that triggers JobHistoryErrorCodeAlertSender. Input DataType: " + e.getClass.getCanonicalName + ) + } + if (!jobHistoryList.isInstanceOf[util.List[_]] || null == jobHistoryList) { + throw new AnomalyScannerException( + 21304, + "Wrong input for JobHistoryErrorCodeAlertSender. Input DataType: " + jobHistoryList.getClass.getCanonicalName + ) + } + for (a <- jobHistoryList.asInstanceOf[util.List[_]].asScala) { + if (a == null) { + logger.warn("Ignore null input data") + } else if (!a.isInstanceOf[JobHistory]) { + logger.warn("Ignore wrong input data Type : " + a.getClass.getCanonicalName) + } else { + val jobHistory = a.asInstanceOf[JobHistory] + val observeInfoMap = BDPJettyServerHelper.gson.fromJson(jobHistory.getObserveInfo, classOf[java.util.Map[String, String]]) + val extraMap = MapUtils.getMap(observeInfoMap, "extra") + observeInfoMap.put("title", extraMap.get("title").toString + ",任务id：" + jobHistory.getId + ",执行结果 :" + jobHistory.getStatus) + observeInfoMap.put("$detail", extraMap.get("detail").toString + ",执行结果 :" + jobHistory.getStatus) + observeInfoMap.put("$submitUser", jobHistory.getSubmitUser) + observeInfoMap.put("$status", jobHistory.getStatus) + observeInfoMap.put("$id", jobHistory.getId.toString) + observeInfoMap.put("$date", dateFormat.format(new Date())) + var alterSysInfo = "" + if (null != extraMap.get("alterSysInfo")) { + alterSysInfo = extraMap.get("alterSysInfo").toString + } + observeInfoMap.put("$sysid", alterSysInfo) + var alterObject = "" + if (null != extraMap.get("alterObject")) { + alterObject = extraMap.get("alterObject").toString + } + observeInfoMap.put("$object", alterObject) + observeInfoMap.put("$ip", InetAddress.getLocalHost.getHostAddress) + observeInfoMap.remove("taskId") + observeInfoMap.remove("extra") + val alters = MonitorAlertUtils.getAlertsByDss(Constants.JOB_RESULT_IM, observeInfoMap) + PooledImsAlertUtils.addAlert(alters.get("12016")) + } + } + } +} diff --git a/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/app/jobhistory/runtime/CommonRunTimeHitEvent.scala b/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/app/jobhistory/runtime/CommonRunTimeHitEvent.scala new file mode 100644 index 0000000000..641ccc2a7d --- /dev/null +++ b/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/app/jobhistory/runtime/CommonRunTimeHitEvent.scala @@ -0,0 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.monitor.scan.app.jobhistory.runtime + +import org.apache.linkis.monitor.scan.core.ob.SingleObserverEvent + +class CommonRunTimeHitEvent extends SingleObserverEvent diff --git a/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/app/jobhistory/runtime/JobHistoryRunTimeAlertSender.scala b/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/app/jobhistory/runtime/JobHistoryRunTimeAlertSender.scala new file mode 100644 index 0000000000..b9f35ce7c7 --- /dev/null +++ b/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/app/jobhistory/runtime/JobHistoryRunTimeAlertSender.scala @@ -0,0 +1,73 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.monitor.scan.app.jobhistory.runtime + +import java.util + +import org.apache.linkis.common.utils.Logging +import org.apache.linkis.monitor.scan.app.jobhistory.entity.JobHistory +import org.apache.linkis.monitor.scan.app.jobhistory.exception.AnomalyScannerException +import org.apache.linkis.monitor.scan.constants.Constants +import org.apache.linkis.monitor.scan.core.ob.{Event, Observer} +import org.apache.linkis.monitor.scan.utils.alert.ims.{MonitorAlertUtils, PooledImsAlertUtils} + +import scala.collection.JavaConverters._ + +/** + * 对前20分钟内的执行数据进行扫描，对已结束的任务进行判断， + * 1.jobhistory中的parm字段中包含（task.notification.conditions） + * 2.执行任务的结果是（Succeed,Failed,Cancelled,Timeout,ALL）其中任意一个，则触发告警 + * 3.job的结果是已经结束 + * 同时满足上述三个条件即可触发告警 + */ +class JobHistoryRunTimeAlertSender() + extends Observer + with Logging { + + override def update(e: Event, jobHistroyList: scala.Any): Unit = { + if (!e.isInstanceOf[JobHistoryRunTimeHitEvent]) { + throw new AnomalyScannerException( + 21304, + "Wrong event that triggers JobHistoryErrorCodeAlertSender. Input DataType: " + e.getClass.getCanonicalName + ) + } + if (null == jobHistroyList || !jobHistroyList.isInstanceOf[util.List[_]]) { + throw new AnomalyScannerException( + 21304, + "Wrong input for JobHistoryErrorCodeAlertSender. Input DataType: " + jobHistroyList.getClass.getCanonicalName + ) + } + for (a <- jobHistroyList.asInstanceOf[util.List[_]].asScala) { + if (a == null) { + logger.warn("Ignore null input data") + } else if (!a.isInstanceOf[JobHistory]) { + logger.warn("Ignore wrong input data Type : " + a.getClass.getCanonicalName) + } else { + // 您的任务ID 1234 执行完成，最终状态为：成功、失败、取消 + val jobHistory = a.asInstanceOf[JobHistory] + val status = jobHistory.getStatus + val replaceParm: util.HashMap[String, String] = new util.HashMap[String, String] + replaceParm.put("$id", String.valueOf(jobHistory.getId)) + replaceParm.put("$status", status) + replaceParm.put("$alteruser", jobHistory.getSubmitUser) + val alters = MonitorAlertUtils.getAlerts(Constants.JOB_RESULT_IM, replaceParm) + PooledImsAlertUtils.addAlert(alters.get("12015")) + } + } + } +} diff --git a/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/app/jobhistory/runtime/JobHistoryRunTimeHitEvent.scala b/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/app/jobhistory/runtime/JobHistoryRunTimeHitEvent.scala new file mode 100644 index 0000000000..40e837d66c --- /dev/null +++ b/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/app/jobhistory/runtime/JobHistoryRunTimeHitEvent.scala @@ -0,0 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.monitor.scan.app.jobhistory.runtime + +import org.apache.linkis.monitor.scan.core.ob.SingleObserverEvent + +class JobHistoryRunTimeHitEvent extends SingleObserverEvent diff --git a/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/app/jobhistory/runtime/JobHistoryRunTimeRule.scala b/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/app/jobhistory/runtime/JobHistoryRunTimeRule.scala new file mode 100644 index 0000000000..e136ea46d0 --- /dev/null +++ b/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/app/jobhistory/runtime/JobHistoryRunTimeRule.scala @@ -0,0 +1,82 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.monitor.scan.app.jobhistory.runtime + +import org.apache.linkis.common.utils.Logging +import org.apache.linkis.monitor.scan.app.jobhistory.entity.JobHistory +import org.apache.linkis.monitor.scan.app.monitor.until.CacheUtils +import org.apache.linkis.monitor.scan.constants.Constants +import org.apache.linkis.monitor.scan.core.ob.Observer +import org.apache.linkis.monitor.scan.core.pac.{AbstractScanRule, ScannedData} +import org.apache.linkis.protocol.utils.TaskUtils +import org.apache.linkis.server.BDPJettyServerHelper + +import java.util +import scala.collection.JavaConverters._ + + +class JobHistoryRunTimeRule(hitObserver: Observer) + extends AbstractScanRule(event = new JobHistoryRunTimeHitEvent, observer = hitObserver) + with Logging { + private val scanRuleList = CacheUtils.cacheBuilder + + /** + * if data match the pattern, return true and trigger observer should call isMatched() + * + * @param data + * @return + */ + override def triggerIfMatched(data: util.List[ScannedData]): Boolean = { + if (null == data || !getHitEvent.isRegistered) { + logger.error("ScanRule is not bind with an observer. Will not be triggered") + return false + } + val alertData: util.List[JobHistory] = new util.ArrayList[JobHistory]() + for (sd <- data.asScala) { + if (sd != null && sd.getData() != null) { + for (d <- sd.getData().asScala) { + d match { + case jobHistory: JobHistory => + if (Constants.DIRTY_DATA_FINISHED_JOB_STATUS.contains(jobHistory.getStatus.toUpperCase())) { + val parmsMap: util.Map[String, scala.AnyRef] = BDPJettyServerHelper.gson.fromJson(jobHistory.getParams, classOf[util.Map[String, scala.AnyRef]]) + val runtimeMap = TaskUtils.getRuntimeMap(parmsMap) + if (runtimeMap.containsKey("task.notification.conditions") && + Constants.DIRTY_DATA_FINISHED_JOB_STATUS.contains(String.valueOf(runtimeMap.get("task.notification.conditions")).toUpperCase())) { + alertData.add(jobHistory) + } + } else { + logger.warn("Ignored wrong input data Type : " + d + ", " + d.getClass.getCanonicalName) + } + scanRuleList.put("jobHistoryId", jobHistory.getId) + case _ => + } + } + } else { + logger.warn("Ignored null scanned data") + } + } + logger.info("hit " + alertData.size() + " data in one iteration") + if (alertData.size() > 0) { + getHitEvent.notifyObserver(getHitEvent, alertData) + true + } else { + false + } + } + +} diff --git a/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/client/MonitorHTTPClient.scala b/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/client/MonitorHTTPClient.scala new file mode 100644 index 0000000000..8813aba210 --- /dev/null +++ b/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/client/MonitorHTTPClient.scala @@ -0,0 +1,117 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.monitor.scan.client + +import org.apache.linkis.httpclient.authentication.AuthenticationStrategy +import org.apache.linkis.httpclient.dws.authentication.StaticAuthenticationStrategy +import org.apache.linkis.httpclient.dws.config.{DWSClientConfig, DWSClientConfigBuilder} +import org.apache.linkis.httpclient.response.Result +import org.apache.linkis.monitor.scan.request.{EmsListAction, EntranceTaskAction, MonitorResourceAction} +import org.apache.linkis.monitor.scan.response.EntranceTaskResult +import org.apache.linkis.ujes.client.response.EmsListResult + +import java.io.Closeable +import java.util.concurrent.TimeUnit + +abstract class MonitorHTTPClient extends Closeable { + + protected[client] def executeJob(ujesJobAction: MonitorResourceAction): Result + + def list(emsListAction: EmsListAction): EmsListResult = { + executeJob(emsListAction).asInstanceOf[EmsListResult] + } + + def entranList(entranceTaskAction: EntranceTaskAction): EntranceTaskResult = { + executeJob(entranceTaskAction).asInstanceOf[EntranceTaskResult] + } + +} + +object MonitorHTTPClient { + + def apply(clientConfig: DWSClientConfig): MonitorHTTPClient = new MonitorHTTPClientClientImpl( + clientConfig + ) + + def apply(serverUrl: String): MonitorHTTPClient = apply(serverUrl, 30000, 10) + + def apply(serverUrl: String, readTimeout: Int, maxConnection: Int): MonitorHTTPClient = + apply(serverUrl, readTimeout, maxConnection, new StaticAuthenticationStrategy, "v1") + + def apply( + serverUrl: String, + readTimeout: Int, + maxConnection: Int, + authenticationStrategy: AuthenticationStrategy, + dwsVersion: String + ): MonitorHTTPClient = { + val clientConfig = DWSClientConfigBuilder + .newBuilder() + .addServerUrl(serverUrl) + .connectionTimeout(30000) + .discoveryEnabled(false) + .loadbalancerEnabled(false) + .maxConnectionSize(maxConnection) + .retryEnabled(false) + .readTimeout(readTimeout) + .setAuthenticationStrategy(authenticationStrategy) + .setDWSVersion(dwsVersion) + .build() + apply(clientConfig) + } + + def getDiscoveryClient(serverUrl: String): MonitorHTTPClient = + getDiscoveryClient(serverUrl, 30000, 10) + + def getDiscoveryClient( + serverUrl: String, + readTimeout: Int, + maxConnection: Int + ): MonitorHTTPClient = + getDiscoveryClient( + serverUrl, + readTimeout, + maxConnection, + new StaticAuthenticationStrategy, + "v1" + ) + + def getDiscoveryClient( + serverUrl: String, + readTimeout: Int, + maxConnection: Int, + authenticationStrategy: AuthenticationStrategy, + dwsVersion: String + ): MonitorHTTPClient = { + val clientConfig = DWSClientConfigBuilder + .newBuilder() + .addServerUrl(serverUrl) + .connectionTimeout(30000) + .discoveryEnabled(true) + .discoveryFrequency(1, TimeUnit.MINUTES) + .loadbalancerEnabled(true) + .maxConnectionSize(maxConnection) + .retryEnabled(false) + .readTimeout(readTimeout) + .setAuthenticationStrategy(authenticationStrategy) + .setDWSVersion(dwsVersion) + .build() + apply(clientConfig) + } + +} diff --git a/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/client/MonitorHTTPClientClientImpl.scala b/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/client/MonitorHTTPClientClientImpl.scala new file mode 100644 index 0000000000..b2b43d7758 --- /dev/null +++ b/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/client/MonitorHTTPClientClientImpl.scala @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.monitor.scan.client + +import org.apache.linkis.httpclient.dws.DWSHttpClient +import org.apache.linkis.httpclient.dws.config.DWSClientConfig +import org.apache.linkis.httpclient.request.Action +import org.apache.linkis.httpclient.response.Result +import org.apache.linkis.monitor.scan.request.MonitorResourceAction + +class MonitorHTTPClientClientImpl(clientConfig: DWSClientConfig) extends MonitorHTTPClient { + + private val dwsHttpClient = + new DWSHttpClient(clientConfig, "Linkis-MonitorResource-Execution-Thread") + + override protected[client] def executeJob(ujesJobAction: MonitorResourceAction): Result = + ujesJobAction match { + + case action: Action => dwsHttpClient.execute(action) + + } + + override def close(): Unit = dwsHttpClient.close() +} diff --git a/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/client/MonitorResourceClient.scala b/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/client/MonitorResourceClient.scala new file mode 100644 index 0000000000..bf605de28b --- /dev/null +++ b/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/client/MonitorResourceClient.scala @@ -0,0 +1,112 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.monitor.scan.client + +import org.apache.linkis.httpclient.authentication.AuthenticationStrategy +import org.apache.linkis.httpclient.dws.authentication.StaticAuthenticationStrategy +import org.apache.linkis.httpclient.dws.config.{DWSClientConfig, DWSClientConfigBuilder} +import org.apache.linkis.httpclient.response.Result +import org.apache.linkis.monitor.scan.request.{EmsListAction, MonitorResourceAction} +import org.apache.linkis.ujes.client.response.EmsListResult + +import java.io.Closeable +import java.util.concurrent.TimeUnit + +abstract class MonitorResourceClient extends Closeable { + + protected[client] def executeJob(ujesJobAction: MonitorResourceAction): Result + + def list(jobListAction: EmsListAction): EmsListResult = { + executeJob(jobListAction).asInstanceOf[EmsListResult] + } + +} + +object MonitorResourceClient { + + def apply(clientConfig: DWSClientConfig): MonitorResourceClient = new MonitorResourceClientImpl( + clientConfig + ) + + def apply(serverUrl: String): MonitorResourceClient = apply(serverUrl, 30000, 10) + + def apply(serverUrl: String, readTimeout: Int, maxConnection: Int): MonitorResourceClient = + apply(serverUrl, readTimeout, maxConnection, new StaticAuthenticationStrategy, "v1") + + def apply( + serverUrl: String, + readTimeout: Int, + maxConnection: Int, + authenticationStrategy: AuthenticationStrategy, + dwsVersion: String + ): MonitorResourceClient = { + val clientConfig = DWSClientConfigBuilder + .newBuilder() + .addServerUrl(serverUrl) + .connectionTimeout(30000) + .discoveryEnabled(false) + .loadbalancerEnabled(false) + .maxConnectionSize(maxConnection) + .retryEnabled(false) + .readTimeout(readTimeout) + .setAuthenticationStrategy(authenticationStrategy) + .setDWSVersion(dwsVersion) + .build() + apply(clientConfig) + } + + def getDiscoveryClient(serverUrl: String): MonitorResourceClient = + getDiscoveryClient(serverUrl, 30000, 10) + + def getDiscoveryClient( + serverUrl: String, + readTimeout: Int, + maxConnection: Int + ): MonitorResourceClient = + getDiscoveryClient( + serverUrl, + readTimeout, + maxConnection, + new StaticAuthenticationStrategy, + "v1" + ) + + def getDiscoveryClient( + serverUrl: String, + readTimeout: Int, + maxConnection: Int, + authenticationStrategy: AuthenticationStrategy, + dwsVersion: String + ): MonitorResourceClient = { + val clientConfig = DWSClientConfigBuilder + .newBuilder() + .addServerUrl(serverUrl) + .connectionTimeout(30000) + .discoveryEnabled(true) + .discoveryFrequency(1, TimeUnit.MINUTES) + .loadbalancerEnabled(true) + .maxConnectionSize(maxConnection) + .retryEnabled(false) + .readTimeout(readTimeout) + .setAuthenticationStrategy(authenticationStrategy) + .setDWSVersion(dwsVersion) + .build() + apply(clientConfig) + } + +} diff --git a/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/client/MonitorResourceClientImpl.scala b/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/client/MonitorResourceClientImpl.scala new file mode 100644 index 0000000000..8e740a94ea --- /dev/null +++ b/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/client/MonitorResourceClientImpl.scala @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.monitor.scan.client + +import org.apache.linkis.httpclient.dws.DWSHttpClient +import org.apache.linkis.httpclient.dws.config.DWSClientConfig +import org.apache.linkis.httpclient.request.Action +import org.apache.linkis.httpclient.response.Result +import org.apache.linkis.monitor.scan.request.MonitorResourceAction + +class MonitorResourceClientImpl(clientConfig: DWSClientConfig) extends MonitorResourceClient { + + private val dwsHttpClient = + new DWSHttpClient(clientConfig, "Linkis-MonitorResource-Execution-Thread") + + override protected[client] def executeJob(ujesJobAction: MonitorResourceAction): Result = + ujesJobAction match { + + case action: Action => dwsHttpClient.execute(action) + + } + + override def close(): Unit = dwsHttpClient.close() +} diff --git a/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/constants/Constants.scala b/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/constants/Constants.scala new file mode 100644 index 0000000000..f3e6d74c48 --- /dev/null +++ b/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/constants/Constants.scala @@ -0,0 +1,113 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.monitor.scan.constants + +import org.apache.linkis.common.conf.CommonVars + + +object Constants { + + val ALERT_IMS_URL = CommonVars.properties.getProperty( + "wds.linkis.alert.url", + "http://127.0.0.1:10812/ims_data_access/send_alarm.do" + ) + + val ALERT_PROPS_FILE_PATH = CommonVars.properties.getProperty( + "wds.linkis.alert.ims.file.path", + "linkis-et-monitor-ims.properties" + ) + + val ALERT_IMS_MAX_LINES = CommonVars[Int]("wds.linkis.alert.ims.max.lines", 8).getValue + + val SCAN_INTERVALS_SECONDS = + CommonVars[Long]("wds.linkis.errorcode.scanner.interval.seconds", 1 * 60 * 60).getValue + + val MAX_INTERVALS_SECONDS = + CommonVars[Long]("wds.linkis.errorcode.scanner.max.interval.seconds", 1 * 60 * 60).getValue + + val ALERT_SUB_SYSTEM_ID = + CommonVars.properties.getProperty("wds.linkis.alert.ims.sub_system_id", "5435") + + val ALERT_DEFAULT_RECEIVERS = CommonVars.properties + .getProperty("wds.linkis.alert.receiver.default", "") + .split(",") + .toSet[String] + + val SCAN_PREFIX_ERRORCODE = "jobhistory.errorcode." + val SCAN_PREFIX_UNFINISHED_JOBTIME_EXCEED_SEC = "jobhistory.unfinished.time.exceed.sec." + + val SCAN_RULE_UNFINISHED_JOB_STATUS = + "Inited,WaitForRetry,Scheduled,Running".split(",").map(s => s.toUpperCase()) + + val DIRTY_DATA_EUREKA_DELETE_INSTANCE_URL = + CommonVars.apply("wds.linkis.eureka.defaultZone", "http://localhost:20303").getValue + + val DIRTY_DATA_EUREKA_DELETE_PATH = CommonVars + .apply("wds.linkis.dirty.data.eureka.delete.path", "/apps/{springName}/{instance}") + .getValue + + val DIRTY_DATA_UNFINISHED_JOB_STATUS = + "Inited,WaitForRetry,Scheduled,Running".split(",").map(s => s.toUpperCase()) + + val DIRTY_DATA_JOB_TARGET_STATUS = "Cancelled" + + val DIRTY_DATA_ENTRANCE_APPLICATIONNAME = + CommonVars("wds.linkis.entrance.spring.name", "linkis-cg-entrance").getValue + + val MODIFY_DB_DATA_DAYS = CommonVars("wds.linkis.dirty.data.modify.db.days", 1).getValue + val ALERT_RESOURCE_MONITOR = "ecm.resource.monitor.im." + + val LINKIS_API_VERSION: CommonVars[String] = + CommonVars[String]("wds.linkis.bml.api.version", "v1") + + val AUTH_TOKEN_KEY: CommonVars[String] = + CommonVars[String]("wds.linkis.bml.auth.token.key", "Validation-Code") + + val AUTH_TOKEN_VALUE: CommonVars[String] = + CommonVars[String]("wds.linkis.bml.auth.token.value", "BML-AUTH") + + val CONNECTION_MAX_SIZE: CommonVars[Int] = + CommonVars[Int]("wds.linkis.bml.connection.max.size", 10) + + val CONNECTION_TIMEOUT: CommonVars[Int] = + CommonVars[Int]("wds.linkis.bml.connection.timeout", 5 * 60 * 1000) + + val CONNECTION_READ_TIMEOUT: CommonVars[Int] = + CommonVars[Int]("wds.linkis.bml.connection.read.timeout", 10 * 60 * 1000) + + val AUTH_TOKEN_KEY_SHORT_NAME = "tokenKey" + val AUTH_TOKEN_VALUE_SHORT_NAME = "tokenValue" + val CONNECTION_MAX_SIZE_SHORT_NAME = "maxConnection" + val CONNECTION_TIMEOUT_SHORT_NAME = "connectTimeout" + val CONNECTION_READ_TIMEOUT_SHORT_NAME = "readTimeout" + val CLIENT_NAME_SHORT_NAME = "clientName" + val USER_LABEL_MONITOR = "jobhistory.label.monitor.im." + + val USER_LABEL_TENANT: CommonVars[String] = + CommonVars[String]("linkis.monitor.jobhistory.userLabel.tenant", "{}") + + val USER_RESOURCE_MONITOR = "user.mode.monitor.im." + val BML_CLEAR_IM = "bml.clear.monitor.im." + val THREAD_TIME_OUT_IM = "thread.monitor.timeout.im." + val JOB_RESULT_IM = "jobhistory.result.monitor.im." + + val DIRTY_DATA_FINISHED_JOB_STATUS = + "Succeed,Failed,Cancelled,Timeout,ALL".split(",").map(s => s.toUpperCase()) + val DIRTY_DATA_FINISHED_JOB_STATUS_ARRAY = "Succeed,Failed,Cancelled,Timeout".split(",") + +} diff --git a/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/constants/ScanOperatorEnum.scala b/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/constants/ScanOperatorEnum.scala new file mode 100644 index 0000000000..8ff3755747 --- /dev/null +++ b/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/constants/ScanOperatorEnum.scala @@ -0,0 +1,24 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.monitor.scan.constants + + +object ScanOperatorEnum extends Enumeration { + type ScanOperatorEnum = Value + val BML_VERSION, JOB_HISTORY = Value +} diff --git a/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/core/ob/Event.scala b/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/core/ob/Event.scala new file mode 100644 index 0000000000..60322c5814 --- /dev/null +++ b/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/core/ob/Event.scala @@ -0,0 +1,29 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.monitor.scan.core.ob + + +trait Event { + def isRegistered: Boolean + + def register(observer: Observer): Unit + + def unRegister(observer: Observer): Unit + + def notifyObserver(event: Event, message: Any): Unit +} diff --git a/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/core/ob/Observer.scala b/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/core/ob/Observer.scala new file mode 100644 index 0000000000..58849dd06c --- /dev/null +++ b/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/core/ob/Observer.scala @@ -0,0 +1,27 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.monitor.scan.core.ob + + +trait Observer { + + /** + * Observer Pattern + */ + def update(event: Event, msg: Any): Unit +} diff --git a/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/core/ob/SingleObserverEvent.java b/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/core/ob/SingleObserverEvent.java new file mode 100644 index 0000000000..2e7a6127bc --- /dev/null +++ b/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/core/ob/SingleObserverEvent.java @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.monitor.scan.core.ob; + + +public class SingleObserverEvent implements Event { + private Observer observer; + + @Override + public boolean isRegistered() { + return observer != null; + } + + @Override + public void register(Observer observer) { + this.observer = observer; + } + + @Override + public void unRegister(Observer observer) { + this.observer = null; + } + + @Override + public void notifyObserver(Event event, Object message) { + observer.update(event, message); + } +} diff --git a/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/core/pac/AbstractDataFetcher.scala b/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/core/pac/AbstractDataFetcher.scala new file mode 100644 index 0000000000..02ad320c55 --- /dev/null +++ b/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/core/pac/AbstractDataFetcher.scala @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.monitor.scan.core.pac + + +abstract class AbstractDataFetcher(customName: String = "") extends DataFetcher { + + private val name: String = if (!customName.isEmpty) { + customName + } else { + this.getClass.getName + "@" + Integer.toHexString(this.hashCode) + } + + def getName(): String = this.name +} diff --git a/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/core/pac/AbstractScanRule.scala b/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/core/pac/AbstractScanRule.scala new file mode 100644 index 0000000000..8d6762c562 --- /dev/null +++ b/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/core/pac/AbstractScanRule.scala @@ -0,0 +1,52 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.monitor.scan.core.pac + +import org.apache.linkis.common.utils.Logging +import org.apache.linkis.monitor.scan.core.ob.{Event, Observer} +import org.apache.linkis.monitor.scan.core.ob.Observer + + +abstract class AbstractScanRule(customName: String = "", event: Event, observer: Observer) + extends ScanRule + with Logging { + event.register(observer) + + private val name: String = if (!customName.isEmpty) { + customName + } else { + this.getClass.getName + "@" + Integer.toHexString(this.hashCode) + } + + def getName(): String = this.name + + /** + * register an observer to trigger if this rule is matched + * + * @param observer + */ + override def addObserver(observer: Observer): Unit = event.register(observer) + + /** + * return registered event + * + * @return + */ + override def getHitEvent(): Event = event + +} diff --git a/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/core/pac/BaseScannedData.scala b/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/core/pac/BaseScannedData.scala new file mode 100644 index 0000000000..02f32a3fc5 --- /dev/null +++ b/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/core/pac/BaseScannedData.scala @@ -0,0 +1,27 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.monitor.scan.core.pac + +import java.util + + +class BaseScannedData(owner: String, data: util.List[scala.Any]) extends ScannedData { + override def getOwner(): String = this.owner + + override def getData(): util.List[scala.Any] = this.data +} diff --git a/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/core/pac/DataFetcher.scala b/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/core/pac/DataFetcher.scala new file mode 100644 index 0000000000..8ac2467684 --- /dev/null +++ b/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/core/pac/DataFetcher.scala @@ -0,0 +1,40 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.monitor.scan.core.pac + +import java.util + +/** + * ScanOperator should encapsulate lower-level client for accessing data from an arbitrary + * datasource. e.g. if we want to scan a DB table. Then operator should encapsulate a DAO + */ +trait DataFetcher { + def getName(): String + + /** + * get arguments for querying data + * + * @return + */ + def getArgs(): Array[scala.Any] + + /** + * make a query to mysql/hive etc. given args + */ + def getData(): util.List[scala.Any] +} diff --git a/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/core/pac/ScanBuffer.scala b/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/core/pac/ScanBuffer.scala new file mode 100644 index 0000000000..d38c02dea7 --- /dev/null +++ b/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/core/pac/ScanBuffer.scala @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.monitor.scan.core.pac + +import java.util +import java.util.concurrent.LinkedBlockingDeque + + +class ScanBuffer { + val buffer: LinkedBlockingDeque[ScannedData] = new LinkedBlockingDeque[ScannedData] + + def write(data: ScannedData): Unit = buffer.add(data) + + def write(data: util.List[ScannedData]): Unit = buffer.addAll(data) + + def drain(maxSize: Int = -1): util.List[ScannedData] = { + val ret = new util.LinkedList[ScannedData] + val realSize = if (maxSize < 0) { + buffer.size + } else { + maxSize + } + buffer.drainTo(ret, realSize) + return ret + } + + def size(): Int = buffer.size() +} diff --git a/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/core/pac/ScanRule.scala b/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/core/pac/ScanRule.scala new file mode 100644 index 0000000000..8f8c3442bb --- /dev/null +++ b/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/core/pac/ScanRule.scala @@ -0,0 +1,50 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.monitor.scan.core.pac + +import org.apache.linkis.monitor.scan.core.ob.{Event, Observer} +import org.apache.linkis.monitor.scan.core.ob.Observer + +import java.util + +trait ScanRule { + + def getName(): String + + /** + * register an observer to trigger if this rule is matched + * + * @param observer + */ + def addObserver(observer: Observer): Unit + + /** + * return registered event + * + * @return + */ + def getHitEvent(): Event + + /** + * if data match the pattern, return true and trigger observer should call isMatched() + * + * @param data + * @return + */ + def triggerIfMatched(data: util.List[ScannedData]): Boolean +} diff --git a/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/core/pac/ScannedData.scala b/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/core/pac/ScannedData.scala new file mode 100644 index 0000000000..a6914c3e60 --- /dev/null +++ b/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/core/pac/ScannedData.scala @@ -0,0 +1,27 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.monitor.scan.core.pac + +import java.util + + +trait ScannedData { + def getOwner(): String + + def getData(): util.List[scala.Any] +} diff --git a/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/core/scanner/AbstractScanner.scala b/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/core/scanner/AbstractScanner.scala new file mode 100644 index 0000000000..e6939df36e --- /dev/null +++ b/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/core/scanner/AbstractScanner.scala @@ -0,0 +1,168 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.monitor.scan.core.scanner + +import java.util +import java.util.concurrent.CopyOnWriteArrayList +import java.util.concurrent.atomic.AtomicInteger + +import org.apache.linkis.common.utils.Logging +import org.apache.linkis.monitor.scan.app.jobhistory.exception.AnomalyScannerException +import org.apache.linkis.monitor.scan.core.ob.{Event, Observer} +import org.apache.linkis.monitor.scan.core.pac._ + + +abstract class AbstractScanner extends AnomalyScanner with Logging { + private val buffer: ScanBuffer = new ScanBuffer + + private val dataFetcherIdx: AtomicInteger = + new AtomicInteger(0) // mark next fetcher for sequentially produce data + + private val dataFetcherList: CopyOnWriteArrayList[DataFetcher] = + new CopyOnWriteArrayList[DataFetcher] + + private val scanRuleList: CopyOnWriteArrayList[ScanRule] = new CopyOnWriteArrayList[ScanRule] + + /** + * Producer + */ + override def addDataFetcher(fetcher: DataFetcher): Unit = { + if (fetcher != null) { + dataFetcherList.add(fetcher) + } else { + warn("ignore null DataFetcher") + } + } + + override def addDataFetchers(fetchers: util.List[DataFetcher]): Unit = { + if (fetchers != null && fetchers.size != 0) { + dataFetcherList.addAll(fetchers) + } else { + warn("ignore null or empty DataFetcher") + } + } + + override def getDataFetchers: util.List[DataFetcher] = dataFetcherList + + /** + * directly feed data to buffer + */ + override def feedData(data: util.List[ScannedData]): Unit = { + if (data != null && data.size != 0) { + buffer.write(data) + } else { + warn("Fed with null or empty data") + } + } + + /** + * Returns a buffer that allows read/write simultaneously buffer is allowed to be written by + * other thread + */ + override def getBuffer(): ScanBuffer = buffer + + /** + * add rules to scanner + */ + override def addScanRule(rule: ScanRule): Unit = { + if (rule != null) { + scanRuleList.add(rule) + } else { + warn("ignore null ScanRule") + } + } + + override def addScanRules(rules: util.List[ScanRule]): Unit = { + if (rules != null && rules.size != 0) { + scanRuleList.addAll(rules) + } else { + warn("ignore null or empty ScanRule") + } + } + + override def getScanRules(): util.List[ScanRule] = scanRuleList + + /** + * blocking call, scan and analyze until all dataFetchers are accessed once + */ + override def run(): Unit = { + if (dataFetcherList.size() == 0) { + throw new AnomalyScannerException( + 21304, + "attempting to run scanner with empty dataFetchers" + ) + } + if (buffer == null) { + throw new AnomalyScannerException(21304, "attempting to run scanner with null buffer") + } + if (scanRuleList.size == 0) { + throw new AnomalyScannerException(21304, "attempting to run scanner with empty rules") + } + while (dataFetcherIdx.get() < dataFetcherList.size()) { + scanOneIteration() + analyzeOneIteration() + } + } + + /** + * 1. scan data for 1 iteration 2. should be a blocking call 3. see if [[ScanRule]] is matched + * 4. trigger [[Event]] and inform observer + */ + override def scanOneIteration(): Unit = { + val idx = dataFetcherIdx.getAndIncrement() + val fetcher = dataFetcherList.get(idx) + if (fetcher != null) { + val rawData = fetcher.getData() + info("scanned " + rawData.size + " data. Rule: " + fetcher.getName); + if (rawData != null && rawData.size != 0) { + buffer.write(new BaseScannedData(fetcher.getName, rawData)) + } + } else { + warn("ignored null fetcher!!") + } + } + + /** + * 1. should be a blocking call 2. read from [[ScanBuffer]] 2. see if [[ScanRule]] is matched + * 3. trigger [[Observer]] + */ + override def analyzeOneIteration(): Unit = { + val dataToAnalyze = buffer.drain() + if (dataToAnalyze != null && dataToAnalyze.size() != 0) { + for (scanRule: ScanRule <- scanRuleList) { + if (scanRule != null) { + info("analyzing " + dataToAnalyze.size + " data. Rule: " + scanRule.getName) + scanRule.triggerIfMatched(dataToAnalyze) + } else { + warn("found empty or null ScanRule") + } + } + } else { + info("analyzed 0 data.") + } + } + + /** + * 1. should be non-blocking 2. keeps calling scanOneIteration() and analyzeOneIteration() + * utils stop() is called + */ + override def start(): Unit = { + // TODO + } + +} diff --git a/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/core/scanner/AnomalyScanner.scala b/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/core/scanner/AnomalyScanner.scala new file mode 100644 index 0000000000..854ca78c3f --- /dev/null +++ b/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/core/scanner/AnomalyScanner.scala @@ -0,0 +1,95 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.monitor.scan.core.scanner + +import org.apache.linkis.monitor.scan.core.ob.Event +import org.apache.linkis.monitor.scan.core.pac.{DataFetcher, ScanBuffer, ScannedData, ScanRule} +import org.apache.linkis.monitor.scan.core.pac.DataFetcher + +import java.util + +/** + * A Scanner that: + * 1. scan a datasource using [[DataFetcher]], write data into [[ScanBuffer]] 2. read data from + * [[ScanBuffer]] see if [[ScanRule]] is matched 3. trigger [[Event]] in [[ScanRule]] and + * inform observer + */ +trait AnomalyScanner { + + /** + * Producer + */ + def addDataFetcher(dataFetcher: DataFetcher): Unit + + def addDataFetchers(dataFetchers: util.List[DataFetcher]): Unit + + def getDataFetchers: util.List[DataFetcher] + + /** + * directly feed data to buffer + */ + def feedData(data: util.List[ScannedData]): Unit + + /** + * Buffer + */ + + /** + * add rules to scanner + */ + def addScanRule(rule: ScanRule): Unit + + def addScanRules(rules: util.List[ScanRule]): Unit + + /** + * Consumer + */ + + def getScanRules(): util.List[ScanRule] + + /** + * scan and analyze for 1 iteration + */ + def run(): Unit + + /** + * 1. should be non-blocking 2. keeps calling scan() utils stop() is called + */ + def start(): Unit + + def shutdown(): Unit + + /** + * 1. should be a blocking call 2. call [[DataFetcher]] to read data 3. write result to + * [[ScanBuffer]] + */ + protected def scanOneIteration(): Unit + + /** + * Returns a buffer that allows read/write simultaneously buffer is allowed to be written by + * other thread + */ + protected def getBuffer(): ScanBuffer + + /** + * 1. should be a blocking call 2. read from [[ScanBuffer]] 2. see if [[ScanRule]] is matched + * 3. trigger [[[[org.apache.linkis.tools.core.ob.Observer]]]] + */ + protected def analyzeOneIteration(): Unit + +} diff --git a/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/core/scanner/DefaultScanner.scala b/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/core/scanner/DefaultScanner.scala new file mode 100644 index 0000000000..c56a3f8182 --- /dev/null +++ b/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/core/scanner/DefaultScanner.scala @@ -0,0 +1,29 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.monitor.scan.core.scanner + +import org.apache.linkis.monitor.scan.utils.alert.ims.PooledImsAlertUtils + + +class DefaultScanner extends AbstractScanner { + + override def shutdown(): Unit = { + PooledImsAlertUtils.shutDown(true, -1) + } + +} diff --git a/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/request/EmsListAction.scala b/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/request/EmsListAction.scala new file mode 100644 index 0000000000..f3906c8634 --- /dev/null +++ b/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/request/EmsListAction.scala @@ -0,0 +1,74 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.monitor.scan.request + +import org.apache.linkis.httpclient.request.GetAction + +import org.apache.commons.lang3.StringUtils + +import scala.collection.mutable.ArrayBuffer + +class EmsListAction extends GetAction with MonitorResourceAction { + + override def suffixURLs: Array[String] = Array("linkisManager", "listAllEMs") + +} + +object EmsListAction { + def newBuilder(): Builder = new Builder + + class Builder private[EmsListAction] () { + private var user: String = _ + private var instance: String = _ + private var nodeHealthy: String = _ + private var owner: String = _ + + def setInstance(instance: String): Builder = { + this.instance = instance + this + } + + def setNodeHealthy(nodeHealthy: String): Builder = { + this.nodeHealthy = nodeHealthy + this + } + + def setOwner(owner: String): Builder = { + this.owner = owner + this + } + + def setUser(user: String): Builder = { + this.user = user + this + } + + def build(): EmsListAction = { + val emsListAction = new EmsListAction + if (StringUtils.isNotBlank(instance)) emsListAction.setParameter("instance", instance) + if (StringUtils.isNotBlank(nodeHealthy)) { + emsListAction.setParameter("nodeHealthy", nodeHealthy) + } + if (StringUtils.isNotBlank(owner)) emsListAction.setParameter("owner", owner) + if (StringUtils.isNotBlank(user)) emsListAction.setUser(user) + emsListAction + } + + } + +} diff --git a/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/request/EntranceTaskAction.scala b/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/request/EntranceTaskAction.scala new file mode 100644 index 0000000000..6126a2b04b --- /dev/null +++ b/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/request/EntranceTaskAction.scala @@ -0,0 +1,76 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.monitor.scan.request + +import org.apache.linkis.httpclient.request.GetAction + +import org.apache.commons.lang3.StringUtils + +class EntranceTaskAction extends GetAction with MonitorResourceAction { + override def suffixURLs: Array[String] = Array("entrance/operation/metrics", "taskinfo") +} + +object EntranceTaskAction { + def newBuilder(): Builder = new Builder + + class Builder private[EntranceTaskAction] () { + private var user: String = _ + private var creator: String = _ + private var engineTypeLabel: String = _ + private var instance: String = _ + + def setCreator(creator: String): Builder = { + this.creator = creator + this + } + + def setEngineTypeLabel(engineTypeLabel: String): Builder = { + this.engineTypeLabel = engineTypeLabel + this + } + + def setUser(user: String): Builder = { + this.user = user + this + } + + def setInstance(instance: String): Builder = { + this.instance = instance + this + } + + def build(): EntranceTaskAction = { + val entranceTaskAction = new EntranceTaskAction + if (StringUtils.isNotBlank(creator)) entranceTaskAction.setParameter("creator", creator) + if (StringUtils.isNotBlank(engineTypeLabel)) entranceTaskAction.setParameter("engineTypeLabel", engineTypeLabel) + if (StringUtils.isNotBlank(instance)) entranceTaskAction.setParameter("instance", instance) + if (StringUtils.isNotBlank(user)) { + // hadoop用户应该获取全部用户entrance信息，则无需传user，即可获取全部entrance信息 + if (user.equals("hadoop")) { + entranceTaskAction.setParameter("user", "") + } else { + entranceTaskAction.setParameter("user", user) + } + } + if (StringUtils.isNotBlank(user)) entranceTaskAction.setUser(user) + entranceTaskAction + } + + } + +} diff --git a/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/request/MonitorResourceAction.scala b/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/request/MonitorResourceAction.scala new file mode 100644 index 0000000000..a8661ec4e7 --- /dev/null +++ b/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/request/MonitorResourceAction.scala @@ -0,0 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.monitor.scan.request + +import org.apache.linkis.httpclient.dws.request.DWSHttpAction + +trait MonitorResourceAction extends DWSHttpAction with UserAction diff --git a/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/request/UserAction.scala b/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/request/UserAction.scala new file mode 100644 index 0000000000..7cae916a74 --- /dev/null +++ b/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/request/UserAction.scala @@ -0,0 +1,26 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.monitor.scan.request + +trait UserAction extends org.apache.linkis.httpclient.request.UserAction { + private var user: String = _ + + override def setUser(user: String): Unit = this.user = user + + override def getUser: String = user +} diff --git a/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/response/EntranceTaskResult.scala b/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/response/EntranceTaskResult.scala new file mode 100644 index 0000000000..fa6ed8080e --- /dev/null +++ b/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/response/EntranceTaskResult.scala @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.monitor.scan.response + +import org.apache.linkis.httpclient.dws.annotation.DWSHttpMessageResult +import org.apache.linkis.httpclient.dws.response.DWSResult + +import java.util + +import scala.beans.BeanProperty + +@DWSHttpMessageResult("/api/rest_j/v\\d+/entrance/operation/metrics/taskinfo") +class EntranceTaskResult extends DWSResult { + + @BeanProperty + var tasks: util.ArrayList[util.Map[String, Object]] = _ + + @BeanProperty + var totalPage: Int = _ + +} diff --git a/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/response/MonitorResourceResult.scala b/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/response/MonitorResourceResult.scala new file mode 100644 index 0000000000..3cd9e74d6d --- /dev/null +++ b/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/response/MonitorResourceResult.scala @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.monitor.scan.response + +import org.apache.linkis.httpclient.dws.response.DWSResult +import org.apache.linkis.httpclient.request.UserAction + +trait MonitorResourceResult extends DWSResult with UserAction { + + private var execID: String = _ + + def getExecID: String = execID + + def setExecID(execID: String): Unit = { + this.execID = execID + } + +} diff --git a/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/utils/ScanUtils.java b/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/utils/ScanUtils.java new file mode 100644 index 0000000000..064865821b --- /dev/null +++ b/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/utils/ScanUtils.java @@ -0,0 +1,71 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.monitor.scan.utils; + + +public class ScanUtils { + public static int getNumOfLines(String str) { + if (str == null || str.length() == 0) { + return 0; + } + int lines = 1; + int len = str.length(); + for (int pos = 0; pos < len; pos++) { + char c = str.charAt(pos); + if (c == '\r') { + lines++; + if (pos + 1 < len && str.charAt(pos + 1) == '\n') { + pos++; + } + } else if (c == '\n') { + lines++; + } + } + return lines; + } + + public static int getFirstIndexSkippingLines(String str, Integer lines) { + if (str == null || str.length() == 0 || lines < 0) { + return -1; + } + if (lines == 0) { + return 0; + } + + int curLineIdx = 0; + int len = str.length(); + for (int pos = 0; pos < len; pos++) { + char c = str.charAt(pos); + if (c == '\r') { + curLineIdx++; + if (pos + 1 < len && str.charAt(pos + 1) == '\n') { + pos++; + } + } else if (c == '\n') { + curLineIdx++; + } else { + continue; + } + + if (curLineIdx >= lines) { + return pos + 1; + } + } + return -1; + } +} diff --git a/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/utils/alert/AlertDesc.scala b/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/utils/alert/AlertDesc.scala new file mode 100644 index 0000000000..b1f29530f8 --- /dev/null +++ b/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/utils/alert/AlertDesc.scala @@ -0,0 +1,26 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.monitor.scan.utils.alert + + +trait AlertDesc { + + /** + * define necessary information for an alert e.g. alert title, alert receiver etc. + */ +} diff --git a/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/utils/alert/AlertSender.scala b/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/utils/alert/AlertSender.scala new file mode 100644 index 0000000000..5ae1b960e5 --- /dev/null +++ b/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/utils/alert/AlertSender.scala @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.monitor.scan.utils.alert + + +trait AlertSender { + + /** + * traverse all registered alertActions and send alert + * + * @return + * true if it is a success + */ + def doSendAlert(alertAction: AlertDesc): Boolean +} diff --git a/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/utils/alert/PooledAlertSender.scala b/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/utils/alert/PooledAlertSender.scala new file mode 100644 index 0000000000..10eb367d2a --- /dev/null +++ b/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/utils/alert/PooledAlertSender.scala @@ -0,0 +1,112 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.monitor.scan.utils.alert + +import org.apache.linkis.common.conf.CommonVars +import org.apache.linkis.common.utils.{Logging, Utils} + +import java.util.concurrent.{Future, LinkedBlockingQueue} +import java.util.concurrent.atomic.{AtomicBoolean, AtomicInteger} + + +abstract class PooledAlertSender extends AlertSender with Logging { + private val THREAD_POOL_SIZE = CommonVars[Int]("wds.linkis.alert.pool.size", 5).getValue + + private val alertDescQ: LinkedBlockingQueue[AlertDesc] = + new LinkedBlockingQueue[AlertDesc](1000) + + protected implicit val executors = + Utils.newCachedExecutionContext(THREAD_POOL_SIZE, "alert-pool-thread-", false) + + private val stopped: AtomicBoolean = new AtomicBoolean(false) + private val runningNumber: AtomicInteger = new AtomicInteger(0) + private var future: Future[_] = _ + + /** + * add an alertDesc to queue + * + * @param alertDesc + * should encapsulates every information an alert platform needs for sending an alarm + */ + def addAlertToPool(alertDesc: AlertDesc): Unit = { + alertDescQ.add(alertDesc) + } + + /** + * describes actual actions for sending an alert + * + * @return + * true if it is a success + */ + override def doSendAlert(alertDesc: AlertDesc): Boolean + + def start(): Unit = { + future = Utils.defaultScheduler.submit(new Runnable() { + override def run() { + info("Pooled alert thread started!") + while (!stopped.get) { + executors synchronized { + while (!stopped.get && runningNumber.get >= THREAD_POOL_SIZE) { + info("Pooled alert thread is full, start waiting") + executors.wait() + } + } + info("Pooled alert thread continue processing") + + if (stopped.get && alertDescQ.size() == 0) return + val alertDesc = Utils.tryQuietly(alertDescQ.take) + if (alertDesc == null) return + executors.submit(new Runnable { + override def run() { + runningNumber.addAndGet(1) + Utils.tryAndWarn { + info("sending alert , information: " + alertDesc) + val ok = doSendAlert(alertDesc) + if (!ok) { + warn("Failed to send alert: " + alertDesc) + } else { + info("successfully send alert: " + alertDesc) + } + runningNumber.decrementAndGet + executors synchronized executors.notify + } + } + }) + } + } + }) + } + + def shutdown(waitComplete: Boolean = true, timeoutMs: Long = -1): Unit = { + info("stopping the Pooled alert thread...") + if (waitComplete) { + val startTime = System.currentTimeMillis() + while ( + (alertDescQ.size() > 0 || runningNumber + .get() > 0) && (timeoutMs == -1 || System.currentTimeMillis() - startTime > timeoutMs) + ) { + Utils.tryQuietly(Thread.sleep(5 * 1000L)) + } + } + executors.shutdown + stopped.set(true) + future.cancel(true) + info("Pooled alert thread is stopped") + } + +} diff --git a/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/utils/alert/ims/ImsAlertDesc.scala b/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/utils/alert/ims/ImsAlertDesc.scala new file mode 100644 index 0000000000..6b12da9584 --- /dev/null +++ b/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/utils/alert/ims/ImsAlertDesc.scala @@ -0,0 +1,155 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.monitor.scan.utils.alert.ims + +import org.apache.linkis.monitor.scan.constants.Constants +import org.apache.linkis.monitor.scan.utils.ScanUtils +import org.apache.linkis.monitor.scan.utils.alert.AlertDesc + +import org.apache.commons.collections.CollectionUtils +import org.apache.commons.lang3.StringUtils + +import java.util +import java.util.HashSet + +import scala.collection.JavaConverters._ + +import ImsAlertLevel.ImsAlertLevel +import ImsAlertWay.ImsAlertWay + + +case class ImsAlertDesc( + var subSystemId: String, + var alertTitle: String, + var alertObj: String, + var alertInfo: String, + alertLevel: ImsAlertLevel = ImsAlertLevel.INFO, + alertIp: String, + canRecover: Int = 0, // 默认0，为1时，需要有对应的恢复告警 + alertWays: util.Set[ImsAlertWay] = new HashSet[ImsAlertWay], + var alertReceivers: util.Set[String] = new HashSet[String], + var numHit: Int = 0, + var hitIntervalMs: Long = 0L +) extends AlertDesc { + + override val toString: String = { + val sb = new StringBuilder + sb.append("sub_system_id=").append(subSystemId).append("&alert_title=").append(alertTitle) + if (alertLevel != null) sb.append("&alert_level=").append(alertLevel.toString) + if (StringUtils.isNotEmpty(alertObj)) sb.append("&alert_obj=").append(alertObj) + if (StringUtils.isNotEmpty(alertInfo)) { + sb.append("&alert_info=") + .append(alertInfo) + .append( + "[freq_info] hit " + numHit + " time(s) within " + hitIntervalMs / 1000 / 60 + " mins" + ) + } + if (canRecover == 0 || canRecover == 1) sb.append("&can_recover=").append(canRecover) + if (alertWays != null && alertWays.size > 0) { + sb.append("&alert_way=") + sb.append(alertWays.asScala.map(_.toString).mkString(",")) + } + if (alertReceivers != null && alertReceivers.size > 0) { + sb.append("&alert_reciver=") + sb.append(alertReceivers.asScala.mkString(",")) + } + if (alertIp != null) { + sb.append("&alert_ip=").append(alertIp) + + } + sb.toString + } + + val toMap: Map[String, String] = { + val map = scala.collection.mutable.Map[String, String]() + map += "sub_system_id" -> subSystemId + map += "alert_title" -> alertTitle + if (alertLevel != null) map += "alert_level" -> alertLevel.toString + if (StringUtils.isNotEmpty(alertObj)) map += "alert_obj" -> alertObj + if (StringUtils.isNotEmpty(alertInfo)) { + map += "alert_info" + "[freq_info] hit " + numHit + " time(s) within " + hitIntervalMs / 1000 / 60 + " mins" -> alertInfo + } + if (canRecover == 0 || canRecover == 1) map += "can_recover" -> canRecover.toString + if (alertWays != null && alertWays.size > 0) { + map += "alert_way" -> alertWays.asScala.map(_.toString).mkString(",") + } + if (alertReceivers != null && alertReceivers.size > 0) { + map += "alert_reciver" -> alertReceivers.asScala.mkString(",") + } + map.toMap + } + + val toImsRequest: ImsRequest = { + val params = validate() + val alertEntity = AlertEntity( + params(0).asInstanceOf[String], + params(1).asInstanceOf[String], + params( + 3 + ) + "[freq_info] hit " + numHit + " time(s) within " + hitIntervalMs / 1000 / 60 + " mins", + alertWays.asScala.map(_.toString).mkString(","), + params(4).asInstanceOf[util.Set[String]].asScala.mkString(","), + alertLevel.toString, + params(2).asInstanceOf[String], + canRecover.toString + ) + + val alertEntityList = new util.ArrayList[AlertEntity] + alertEntityList.add(alertEntity) + + ImsRequest(alertEntityList) + } + + def validate(): Array[Any] = { + assert(StringUtils.isNumeric(subSystemId) && subSystemId.length == 4) + assert(StringUtils.isNotEmpty(alertTitle)) + val newAlertTitle = if (alertTitle.length > 100) { + alertTitle.substring(0, 96) + "... ..." + } else { + alertTitle + } + val newAlertObj = if (StringUtils.isNotEmpty(alertObj) && alertObj.length >= 50) { + alertObj = alertObj.substring(0, 36) + "... ..." + } else { + alertObj + } + val newAlertInfo = + if ( + StringUtils.isNotEmpty(alertInfo) && ScanUtils.getNumOfLines( + alertInfo + ) > Constants.ALERT_IMS_MAX_LINES + ) { + StringUtils.substring( + alertInfo, + 0, + ScanUtils.getFirstIndexSkippingLines(alertInfo, Constants.ALERT_IMS_MAX_LINES) + ) + "... ...\n" + } else { + alertInfo + } + val newAlertReceivers = + if (CollectionUtils.isNotEmpty(alertReceivers) && alertReceivers.size > 15) { + alertReceivers.asScala.take(15) + } else { + alertReceivers + } + + Array(subSystemId, newAlertTitle, newAlertObj, newAlertInfo, newAlertReceivers) + } + +} diff --git a/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/utils/alert/ims/ImsAlertLevel.scala b/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/utils/alert/ims/ImsAlertLevel.scala new file mode 100644 index 0000000000..cb304e9e4f --- /dev/null +++ b/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/utils/alert/ims/ImsAlertLevel.scala @@ -0,0 +1,29 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.monitor.scan.utils.alert.ims + + +object ImsAlertLevel extends Enumeration { + type ImsAlertLevel = Value + val INFO = Value("5") + val WARN = Value("4") + val MINOR = Value("3") + val MAJOR = Value("2") + val CRITICAL = Value("1") + val CLEAR = Value("0") +} diff --git a/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/utils/alert/ims/ImsAlertWay.scala b/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/utils/alert/ims/ImsAlertWay.scala new file mode 100644 index 0000000000..207df613d7 --- /dev/null +++ b/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/utils/alert/ims/ImsAlertWay.scala @@ -0,0 +1,27 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.monitor.scan.utils.alert.ims + + +object ImsAlertWay extends Enumeration { + type ImsAlertWay = Value + val NoAlert = Value("0") + val RTX = Value("1") + val Email = Value("2") + val WeChat = Value("3") +} diff --git a/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/utils/alert/ims/ImsRequest.scala b/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/utils/alert/ims/ImsRequest.scala new file mode 100644 index 0000000000..64b80d3a95 --- /dev/null +++ b/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/utils/alert/ims/ImsRequest.scala @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.monitor.scan.utils.alert.ims + +import java.util + +import com.fasterxml.jackson.annotation.JsonProperty + + +case class ImsRequest(@JsonProperty("alertList") alertList: util.List[AlertEntity]) + +case class AlertEntity( + @JsonProperty("sub_system_id") subSystemId: String, + @JsonProperty("alert_title") alertTitle: String, + @JsonProperty("alert_info") alertInfo: String, + @JsonProperty("alert_way") alertWays: String, + @JsonProperty("alert_reciver") alertReceivers: String, + @JsonProperty("alert_level") alertLevel: String, + @JsonProperty("alert_obj") alertObj: String, + @JsonProperty("can_recover") canRecover: String +) diff --git a/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/utils/alert/ims/JobHistoryScanImsAlertPropFileParserUtils.scala b/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/utils/alert/ims/JobHistoryScanImsAlertPropFileParserUtils.scala new file mode 100644 index 0000000000..95b64f3f4e --- /dev/null +++ b/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/utils/alert/ims/JobHistoryScanImsAlertPropFileParserUtils.scala @@ -0,0 +1,132 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.monitor.scan.utils.alert.ims + +import org.apache.linkis.common.utils.{JsonUtils, Logging, Utils} +import org.apache.linkis.monitor.scan.app.jobhistory.exception.AnomalyScannerException +import org.apache.linkis.monitor.scan.constants.Constants +import org.apache.linkis.monitor.scan.utils.alert.AlertDesc +import org.apache.commons.lang3.StringUtils +import org.apache.commons.lang3.exception.ExceptionUtils + +import java.io.{BufferedReader, File, FileInputStream, InputStream, InputStreamReader} +import java.text.SimpleDateFormat +import java.util +import java.util.Properties +import scala.collection.JavaConverters._ +import com.fasterxml.jackson.annotation.JsonProperty +import com.fasterxml.jackson.databind.{DeserializationFeature, ObjectMapper} +import com.fasterxml.jackson.module.scala.DefaultScalaModule +import org.apache.commons.io.IOUtils + + +object JobHistoryScanImsAlertPropFileParserUtils extends Logging { + + private val mapper = { + val ret = new ObjectMapper().setDateFormat(new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ssZ")) + ret.registerModule(DefaultScalaModule) + ret.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false) + ret + } + + def getAlerts(prefix: String): util.Map[String, AlertDesc] = { + val ret = new util.HashMap[String, AlertDesc]() + val url = getClass.getClassLoader.getResource(Constants.ALERT_PROPS_FILE_PATH) + if (url == null) { + throw new AnomalyScannerException( + 21304, + "Failed to load alerts from alert properties. Alert properties file does not exist: " + Constants.ALERT_PROPS_FILE_PATH + ) + } + logger.info("reading alert properties from: " + url.getFile) + val properties = new Properties() + var inputStream: InputStream = null + var reader: InputStreamReader = null + var buff: BufferedReader = null + + Utils.tryFinally { + Utils.tryCatch { + inputStream = new FileInputStream(new File(url.getFile)) + reader = new InputStreamReader(inputStream, "UTF-8") + buff = new BufferedReader(reader) + properties.load(buff) + } { t => { + throw new AnomalyScannerException(21304, "Failed to load alerts from alert properties. Cause: " + ExceptionUtils.getMessage(t)) + return ret + } + } + } { + IOUtils.closeQuietly(buff) + IOUtils.closeQuietly(reader) + IOUtils.closeQuietly(inputStream) + } + for ((k: String, v: String) <- properties.asScala) { + if (ret.containsKey(k)) { + logger.warn("found duplicate key in alert properties, accept only the first one") + } else if (StringUtils.startsWith(k, prefix)) { + val data = mapper.readValue(v, classOf[ImsAlertPropFileData]) + val receivers = { + val set: util.Set[String] = new util.HashSet[String] + if (StringUtils.isNotBlank(data.alertReceivers)) { + data.alertReceivers.split(",").map(r => set.add(r)) + } + Constants.ALERT_DEFAULT_RECEIVERS.foreach(e => { + if (StringUtils.isNotBlank(e)) { + set.add(e) + } + }) + set + } + val alertDesc = Utils.tryAndWarn( + new ImsAlertDesc( + Constants.ALERT_SUB_SYSTEM_ID, + data.alertTitle, + data.alertObj, + data.alertInfo, + ImsAlertLevel.withName(data.alertLevel), + null, // Not used + 0, { + val set: util.Set[ImsAlertWay.Value] = new util.HashSet[ImsAlertWay.Value] + if (StringUtils.isNotBlank(data.alertWays)) { + data.alertWays + .split(",") + .map(alertWayStr => set.add(ImsAlertWay.withName(alertWayStr))) + } + set + }, + receivers + ) + ) + val realK = StringUtils.substringAfter(k, prefix) + ret.put(realK, alertDesc) + } + } + ret + } + +} + +case class ImsAlertPropFileData( + @JsonProperty("alert_title") alertTitle: String, + @JsonProperty("alert_info") alertInfo: String, + @JsonProperty("alert_way") alertWays: String, + @JsonProperty("alert_reciver") alertReceivers: String, + @JsonProperty("alert_level") alertLevel: String, + @JsonProperty("alert_obj") alertObj: String, + @JsonProperty("can_recover") canRecover: String +) diff --git a/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/utils/alert/ims/MonitorAlertUtils.scala b/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/utils/alert/ims/MonitorAlertUtils.scala new file mode 100644 index 0000000000..021ec3b588 --- /dev/null +++ b/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/utils/alert/ims/MonitorAlertUtils.scala @@ -0,0 +1,229 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.monitor.scan.utils.alert.ims + +import org.apache.linkis.common.utils.{JsonUtils, Logging, Utils} +import org.apache.linkis.monitor.scan.app.jobhistory.exception.AnomalyScannerException +import org.apache.linkis.monitor.scan.constants.Constants +import org.apache.linkis.monitor.scan.utils.alert.AlertDesc +import org.apache.commons.lang3.StringUtils +import org.apache.commons.lang3.exception.ExceptionUtils + +import java.io.{BufferedReader, File, FileInputStream, InputStream, InputStreamReader} +import java.text.SimpleDateFormat +import java.util +import java.util.Properties +import scala.collection.JavaConverters._ +import com.fasterxml.jackson.databind.{DeserializationFeature, ObjectMapper} +import com.fasterxml.jackson.module.scala.DefaultScalaModule +import org.apache.commons.io.IOUtils + +object MonitorAlertUtils extends Logging { + + + private val mapper = { + val ret = new ObjectMapper().setDateFormat(new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ssZ")) + ret.registerModule(DefaultScalaModule) + ret.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false) + ret + } + + def getAlerts( + prefix: String, + parms: util.HashMap[String, String] + ): util.Map[String, AlertDesc] = { + val ret = new util.HashMap[String, AlertDesc]() + val url = getClass.getClassLoader.getResource(Constants.ALERT_PROPS_FILE_PATH) + if (url == null) { + throw new AnomalyScannerException( + 21304, + "Failed to load alerts from alert properties. Alert properties file does not exist: " + Constants.ALERT_PROPS_FILE_PATH + ) + } + logger.info("reading alert properties from: " + url.getFile) + val properties = new Properties() + var inputStream: InputStream = null + + var reader: InputStreamReader = null + var buff: BufferedReader = null + + Utils.tryFinally { + Utils.tryThrow { + inputStream = new FileInputStream(new File(url.getFile)) + reader = new InputStreamReader(inputStream, "UTF-8") + buff = new BufferedReader(reader) + properties.load(buff) + } { + case t: Throwable => + new AnomalyScannerException( + 21304, + "Failed to load alerts from alert properties. Cause: " + ExceptionUtils.getMessage(t) + ) + } + } { + IOUtils.closeQuietly(buff) + IOUtils.closeQuietly(reader) + IOUtils.closeQuietly(inputStream) + } + for ((k: String, v: String) <- properties.asScala) { + if (ret.containsKey(k)) { + logger.warn("found duplicate key in alert properties, accept only the first one") + } else if (StringUtils.startsWith(k, prefix)) { + val data = mapper.readValue(v, classOf[ImsAlertPropFileData]) + var alertInfo = new String( + new StringBuilder().append(data.alertInfo).toString().getBytes(), + "utf-8" + ).replace("$name", data.alertReceivers) + val interator = parms.keySet.iterator + while (interator.hasNext) { + val key = interator.next + val value = parms.get(key) + alertInfo = alertInfo.replace(key, value) + } + val receivers = { + val set: util.Set[String] = new util.HashSet[String] + if (StringUtils.isNotBlank(data.alertReceivers)) { + data.alertReceivers.split(",").map(r => set.add(r)) + } + Constants.ALERT_DEFAULT_RECEIVERS.foreach(e => { + if (StringUtils.isNotBlank(e)) { + set.add(e) + } + }) + set.add(parms.get("$alteruser")) + set + } + val alertDesc = Utils.tryAndWarn( + new ImsAlertDesc( + Constants.ALERT_SUB_SYSTEM_ID, + data.alertTitle, + data.alertObj, + alertInfo, + ImsAlertLevel.withName(data.alertLevel), + null, // Not used + 0, { + val set: util.Set[ImsAlertWay.Value] = new util.HashSet[ImsAlertWay.Value] + if (StringUtils.isNotBlank(data.alertWays)) { + data.alertWays + .split(",") + .map(alertWayStr => set.add(ImsAlertWay.withName(alertWayStr))) + } + set + }, + receivers + ) + ) + val realK = StringUtils.substringAfter(k, prefix) + ret.put(realK, alertDesc) + } + } + ret + } + + + def getAlertsByDss(prefix: String, parms: util.Map[String, String]): util.Map[String, AlertDesc] = { + val ret = new util.HashMap[String, AlertDesc]() + val url = getClass.getClassLoader.getResource(Constants.ALERT_PROPS_FILE_PATH) + if (url == null) { + throw new AnomalyScannerException( + 21304, + "Failed to load alerts from alert properties. Alert properties file does not exist: " + Constants.ALERT_PROPS_FILE_PATH + ) + } + logger.info("reading alert properties from: " + url.getFile) + val properties = new Properties() + var inputStream: InputStream = null + + var reader: InputStreamReader = null + var buff: BufferedReader = null + + Utils.tryFinally { + Utils.tryThrow{ + inputStream = new FileInputStream(new File(url.getFile)) + reader = new InputStreamReader(inputStream, "UTF-8") + buff = new BufferedReader(reader) + properties.load(buff) + } { + case t: Throwable => + new AnomalyScannerException( + 21304, + "Failed to load alerts from alert properties. Cause: " + ExceptionUtils.getMessage(t) + ) + } + } { + IOUtils.closeQuietly(buff) + IOUtils.closeQuietly(reader) + IOUtils.closeQuietly(inputStream) + } + for ((k: String, v: String) <- properties.asScala) { + if (ret.containsKey(k)) { + warn("found duplicate key in alert properties, accept only the first one") + } else if (StringUtils.startsWith(k, prefix)) { + val data = mapper.readValue(v, classOf[ImsAlertPropFileData]) + var alertInfo = data.alertInfo + val interator = parms.keySet.iterator + while (interator.hasNext) { + val key = interator.next + val value = parms.get(key) + alertInfo = alertInfo.replace(key, value) + } +// alertInfo = parms.getOrDefault("detail", "").concat(alertInfo) + val receivers = { + val set: util.Set[String] = new util.HashSet[String] + if (StringUtils.isNotBlank(data.alertReceivers)) { + data.alertReceivers.split(",").map(r => set.add(r)) + } + Constants.ALERT_DEFAULT_RECEIVERS.foreach(e => { + if (StringUtils.isNotBlank(e)) { + set.add(e) + } + }) + if (StringUtils.isNotBlank(parms.get("receiver"))) { + parms.get("receiver").split(",").map(r => set.add(r)) + } + set + } + val alertDesc = Utils.tryAndWarn( + ImsAlertDesc( + parms.getOrDefault("subSystemId", Constants.ALERT_SUB_SYSTEM_ID), + parms.getOrDefault("title", ""), + data.alertObj, + alertInfo + , + ImsAlertLevel.withName(parms.getOrDefault("monitorLevel", "4")), + null, // Not used + 0, { + val set: util.Set[ImsAlertWay.Value] = new util.HashSet[ImsAlertWay.Value] + if (StringUtils.isNotBlank(data.alertWays)) { + data.alertWays + .split(",") + .map(alertWayStr => set.add(ImsAlertWay.withName(alertWayStr))) + } + set + }, + receivers + ) + ) + val realK = StringUtils.substringAfter(k, prefix) + ret.put(realK, alertDesc) + } + } + ret + } + +} diff --git a/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/utils/alert/ims/PooledImsAlertSender.scala b/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/utils/alert/ims/PooledImsAlertSender.scala new file mode 100644 index 0000000000..d2508b37a3 --- /dev/null +++ b/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/utils/alert/ims/PooledImsAlertSender.scala @@ -0,0 +1,110 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.monitor.scan.utils.alert.ims + +import org.apache.linkis.common.utils.{JsonUtils, Logging, Utils} +import org.apache.linkis.monitor.scan.utils.alert.{AlertDesc, PooledAlertSender} +import org.apache.linkis.monitor.scan.utils.alert.AlertDesc +import org.apache.linkis.monitor.scan.utils.log.LogUtils + +import org.apache.http.client.config.RequestConfig +import org.apache.http.client.methods.HttpPost +import org.apache.http.entity.{ContentType, StringEntity} +import org.apache.http.impl.client.HttpClients +import org.apache.http.util.EntityUtils + +import java.text.SimpleDateFormat +import java.util + +import com.fasterxml.jackson.databind.ObjectMapper +import com.fasterxml.jackson.module.scala.DefaultScalaModule + + +class PooledImsAlertSender( + subSystemId: String, + alertUrl: String, + default_Receivers: util.Set[String] +) extends PooledAlertSender + with Logging { + + protected val httpClient = HttpClients.createDefault // TODO: Linkis-httpclient + + private val mapper = + new ObjectMapper().setDateFormat(new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ssZ")) + + /** + * describes actual actions for sending an alert + * + * @return + * true if it is a success + */ + + /** + * describes actual actions for sending an alert + * + * @return + * true if it is a success + */ + override def doSendAlert(alertDesc: AlertDesc): Boolean = { + if (!alertDesc.isInstanceOf[ImsAlertDesc]) { + logger.warn("wrong alertDesc dataType: " + alertDesc.getClass.getCanonicalName) + return false + } + logger.info("sending an alert to IMS, information: " + alertDesc) + val imsRequest = alertDesc.asInstanceOf[ImsAlertDesc].toImsRequest + + mapper.registerModule(DefaultScalaModule) + val paramContent = Utils.tryCatch(mapper.writeValueAsString(imsRequest)) { t => + logger.warn("ignore alert: " + imsRequest, t) + return false + } + if (paramContent.isEmpty) { + logger. warn("alertParams is empty, will not send alarm") + return false + } + + val requestConfig = RequestConfig.DEFAULT + + val entity = new StringEntity( + paramContent, + ContentType.create(ContentType.APPLICATION_JSON.getMimeType, "UTF-8") + ) + entity.setContentEncoding("UTF-8") + + val httpPost = new HttpPost(alertUrl) + + httpPost.setConfig(requestConfig) + httpPost.setEntity(entity) + + val response = Utils.tryAndErrorMsg(httpClient.execute(httpPost))("send alert to IMS failed") + + if (response != null) { + val responseInfo = EntityUtils.toString(response.getEntity, "UTF-8") + logger.info("Alert: " + paramContent + "Response: " + responseInfo) + LogUtils.stdOutLogger.info("Alert: " + paramContent + "Response: " + responseInfo) + if (response.getStatusLine.getStatusCode == 200) return true + } + return false + } + + override def shutdown(waitComplete: Boolean = true, timeoutMs: Long = -1): Unit = { + super.shutdown(waitComplete, timeoutMs) + httpClient.close + } + +} diff --git a/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/utils/alert/ims/PooledImsAlertUtils.scala b/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/utils/alert/ims/PooledImsAlertUtils.scala new file mode 100644 index 0000000000..4a50161438 --- /dev/null +++ b/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/utils/alert/ims/PooledImsAlertUtils.scala @@ -0,0 +1,115 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.monitor.scan.utils.alert.ims + +import org.apache.linkis.common.utils.{Logging, Utils} +import org.apache.linkis.monitor.scan.constants.Constants +import org.apache.linkis.monitor.scan.utils.alert.AlertDesc + +import org.apache.commons.collections.CollectionUtils +import org.apache.commons.lang3.StringUtils +import org.apache.commons.lang3.exception.ExceptionUtils + +import java.net.InetAddress +import java.util +import java.util.HashSet + +import scala.collection.JavaConverters._ + +import ImsAlertWay.ImsAlertWay + + +object PooledImsAlertUtils extends Logging { + + private val sender: PooledImsAlertSender = { + val ret = new PooledImsAlertSender( + Constants.ALERT_SUB_SYSTEM_ID, + Constants.ALERT_IMS_URL, + Constants.ALERT_DEFAULT_RECEIVERS.asJava + ) + ret.start() + ret + } + + private val localIp = InetAddress.getLocalHost.getHostAddress + + def addAlertAndLogException(message: String): Unit = Utils.tryAndError(addAlert(message)) + + def addAlert(message: String): Unit = addExceptionAlert(message, null, null) + + def addExceptionAlert(message: String, t: Throwable): Unit = + addExceptionAlert(message, t, null) + + def addExceptionAlertAndLogException(message: String, t: Throwable): Unit = + Utils.tryAndError(addExceptionAlert(message, t, null)) + + def addExceptionAlert(message: String, t: Throwable, alertWays: util.Set[ImsAlertWay]): Unit = { + val alertObj = + if (StringUtils.isEmpty(message) && t != null) t.getMessage + else if (StringUtils.isEmpty(message)) { + throw new NullPointerException("both message and exception are null!") + } else { + message + } + val _alertWays = + if (CollectionUtils.isNotEmpty(alertWays)) alertWays else new HashSet[ImsAlertWay]() + val (alertInfo, alertLevel) = if (t != null) { + _alertWays.add(ImsAlertWay.Email) + _alertWays.add(ImsAlertWay.RTX) + _alertWays.add(ImsAlertWay.WeChat) + (ExceptionUtils.getRootCauseMessage(t), ImsAlertLevel.MAJOR) + } else { + _alertWays.add(ImsAlertWay.RTX) + (message, ImsAlertLevel.WARN) + } + val alertDesc = new ImsAlertDesc( + Constants.ALERT_SUB_SYSTEM_ID, + "BDP Alert", + alertObj, + alertInfo, + alertLevel, + localIp, + 0, + _alertWays + ) + addAlert(alertDesc) + } + + def addAlert(alertDesc: AlertDesc): Unit = { + if (!alertDesc.isInstanceOf[ImsAlertDesc]) { + logger.warn("Ignore wrong alertDesc. DataType: " + alertDesc.getClass.getCanonicalName) + } else { + sender.addAlertToPool(alertDesc) + logger.info("successfully added alert") + } + } + + def addAlertAndLogException(alertDesc: ImsAlertDesc): Unit = + Utils.tryAndError(addAlert(alertDesc)) + + def clearAlert(alertDesc: ImsAlertDesc): Unit = { + assert(alertDesc.canRecover == 1) + assert(alertDesc.alertLevel == ImsAlertLevel.CLEAR) + sender.addAlertToPool(alertDesc) + } + + def shutDown(waitComplete: Boolean = true, timeoutMs: Long = -1): Unit = { + sender.shutdown(waitComplete, timeoutMs) + } + +} diff --git a/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/utils/alert/ims/UserLabelAlertUtils.scala b/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/utils/alert/ims/UserLabelAlertUtils.scala new file mode 100644 index 0000000000..1083967783 --- /dev/null +++ b/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/utils/alert/ims/UserLabelAlertUtils.scala @@ -0,0 +1,126 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.monitor.scan.utils.alert.ims + +import org.apache.linkis.common.utils.{JsonUtils, Logging, Utils} +import org.apache.linkis.monitor.scan.app.jobhistory.exception.AnomalyScannerException +import org.apache.linkis.monitor.scan.constants.Constants +import org.apache.linkis.monitor.scan.utils.alert.AlertDesc +import org.apache.commons.lang3.StringUtils +import org.apache.commons.lang3.exception.ExceptionUtils + +import java.io.{BufferedReader, File, FileInputStream, InputStream, InputStreamReader} +import java.text.SimpleDateFormat +import java.util +import java.util.Properties +import scala.collection.JavaConverters._ +import com.fasterxml.jackson.databind.{DeserializationFeature, ObjectMapper} +import com.fasterxml.jackson.module.scala.DefaultScalaModule +import org.apache.commons.io.IOUtils + +object UserLabelAlertUtils extends Logging { + + private val mapper = { + val ret = new ObjectMapper().setDateFormat(new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ssZ")) + ret.registerModule(DefaultScalaModule) + ret.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false) + ret + } + + def getAlerts(prefix: String, instans: String): util.Map[String, AlertDesc] = { + val ret = new util.HashMap[String, AlertDesc]() + val url = getClass.getClassLoader.getResource(Constants.ALERT_PROPS_FILE_PATH) + if (url == null) { + throw new AnomalyScannerException( + 21304, + "Failed to load alerts from alert properties. Alert properties file does not exist: " + Constants.ALERT_PROPS_FILE_PATH + ) + } + logger.info("reading alert properties from: " + url.getFile) + val properties = new Properties() + var inputStream: InputStream = null + + var reader: InputStreamReader = null + var buff: BufferedReader = null + + Utils.tryFinally { + Utils.tryThrow { + inputStream = new FileInputStream(new File(url.getFile)) + reader = new InputStreamReader(inputStream, "UTF-8") + buff = new BufferedReader(reader) + properties.load(buff) + } { + case t: Throwable => + new AnomalyScannerException( + 21304, + "Failed to load alerts from alert properties. Cause: " + ExceptionUtils.getMessage(t) + ) + } + } { + IOUtils.closeQuietly(buff) + IOUtils.closeQuietly(reader) + IOUtils.closeQuietly(inputStream) + } + for ((k: String, v: String) <- properties.asScala) { + if (ret.containsKey(k)) { + logger.warn("found duplicate key in alert properties, accept only the first one") + } else if (StringUtils.startsWith(k, prefix)) { + val data = mapper.readValue(v, classOf[ImsAlertPropFileData]) + val alertInfo = + new String(new StringBuilder().append(data.alertInfo).toString().getBytes(), "utf-8") + .replace("$userCreator", instans) + val receivers = { + val set: util.Set[String] = new util.HashSet[String] + if (StringUtils.isNotBlank(data.alertReceivers)) { + data.alertReceivers.split(",").map(r => set.add(r)) + } + Constants.ALERT_DEFAULT_RECEIVERS.foreach(e => { + if (StringUtils.isNotBlank(e)) { + set.add(e) + } + }) + set + } + val alertDesc = Utils.tryAndWarn( + new ImsAlertDesc( + Constants.ALERT_SUB_SYSTEM_ID, + data.alertTitle, + data.alertObj, + alertInfo, + ImsAlertLevel.withName(data.alertLevel), + null, // Not used + 0, { + val set: util.Set[ImsAlertWay.Value] = new util.HashSet[ImsAlertWay.Value] + if (StringUtils.isNotBlank(data.alertWays)) { + data.alertWays + .split(",") + .map(alertWayStr => set.add(ImsAlertWay.withName(alertWayStr))) + } + set + }, + receivers + ) + ) + val realK = StringUtils.substringAfter(k, prefix) + ret.put(realK, alertDesc) + } + } + ret + } + +} diff --git a/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/utils/log/LogUtils.scala b/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/utils/log/LogUtils.scala new file mode 100644 index 0000000000..38af7b6104 --- /dev/null +++ b/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/utils/log/LogUtils.scala @@ -0,0 +1,25 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.monitor.scan.utils.log + +import org.slf4j.LoggerFactory + + +object LogUtils { + val stdOutLogger = LoggerFactory.getLogger("PlaintTextConsoleLogger") +} diff --git a/linkis-extensions/linkis-et-monitor/src/test/java/org/apache/linkis/monitor/scan/app/JobHistoryScanImsAlertPropFileParserUtilsTest.java b/linkis-extensions/linkis-et-monitor/src/test/java/org/apache/linkis/monitor/scan/app/JobHistoryScanImsAlertPropFileParserUtilsTest.java new file mode 100644 index 0000000000..600ccafddb --- /dev/null +++ b/linkis-extensions/linkis-et-monitor/src/test/java/org/apache/linkis/monitor/scan/app/JobHistoryScanImsAlertPropFileParserUtilsTest.java @@ -0,0 +1,48 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.monitor.scan.app; + +import org.apache.linkis.monitor.scan.constants.Constants; +import org.apache.linkis.monitor.scan.utils.alert.AlertDesc; +import org.apache.linkis.monitor.scan.utils.alert.ims.JobHistoryScanImsAlertPropFileParserUtils; +import org.apache.linkis.server.utils.LinkisMainHelper; + +import java.util.Map; + +import org.junit.Assert; + +public class JobHistoryScanImsAlertPropFileParserUtilsTest { + // @Before + public void before() { + System.getProperties().setProperty(LinkisMainHelper.SERVER_NAME_KEY(), "linkis-et-monitor"); + System.getProperties() + .setProperty("log4j.configurationFile", "src/test/resources/log4j2-console.xml"); + // System.getProperties().setProperty("wds.linkis.server.conf", + // "linkis-et-monitor.properties"); + } + + // @Test + public void getAlerts() throws Exception { + Map alerts = + JobHistoryScanImsAlertPropFileParserUtils.getAlerts(Constants.SCAN_PREFIX_ERRORCODE()); + for (Map.Entry kv : alerts.entrySet()) { + System.out.println(kv.getKey() + ": " + kv.getValue().toString()); + } + Assert.assertEquals(alerts.size(), 2); + } +} diff --git a/linkis-extensions/linkis-et-monitor/src/test/java/org/apache/linkis/monitor/scan/app/LinkisJobHistoryScanApplicationTest.java b/linkis-extensions/linkis-et-monitor/src/test/java/org/apache/linkis/monitor/scan/app/LinkisJobHistoryScanApplicationTest.java new file mode 100644 index 0000000000..716ad5cf43 --- /dev/null +++ b/linkis-extensions/linkis-et-monitor/src/test/java/org/apache/linkis/monitor/scan/app/LinkisJobHistoryScanApplicationTest.java @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.monitor.scan.app; + +import org.apache.linkis.server.utils.LinkisMainHelper; + + +public class LinkisJobHistoryScanApplicationTest { + // @Before + public void before() { + System.getProperties().setProperty(LinkisMainHelper.SERVER_NAME_KEY(), "linkis-et-monitor"); + System.getProperties() + .setProperty("log4j.configurationFile", "src/test/resources/log4j2-console.xml"); + // System.getProperties().setProperty("wds.linkis.server.conf", + // "linkis-et-monitor.properties"); + } + + // @Test + public void main() throws Exception { + LinkisJobHistoryScanApplication.main(new String[] {}); + // LinkisJobHistoryScanApplication.main(new String[]{"2021122919", "2021122921"}); + } +} diff --git a/linkis-extensions/linkis-et-monitor/src/test/java/org/apache/linkis/monitor/scan/utils/alert/PooledImsAlertSenderTest.java b/linkis-extensions/linkis-et-monitor/src/test/java/org/apache/linkis/monitor/scan/utils/alert/PooledImsAlertSenderTest.java new file mode 100644 index 0000000000..8cfad4c989 --- /dev/null +++ b/linkis-extensions/linkis-et-monitor/src/test/java/org/apache/linkis/monitor/scan/utils/alert/PooledImsAlertSenderTest.java @@ -0,0 +1,67 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.monitor.scan.utils.alert; + +import org.apache.linkis.monitor.scan.utils.alert.ims.ImsAlertDesc; +import org.apache.linkis.monitor.scan.utils.alert.ims.ImsAlertLevel; +import org.apache.linkis.monitor.scan.utils.alert.ims.ImsAlertWay; +import org.apache.linkis.monitor.scan.utils.alert.ims.PooledImsAlertSender; +import org.apache.linkis.server.utils.LinkisMainHelper; + +import java.util.HashSet; +import java.util.Set; + +public class PooledImsAlertSenderTest { + // @Before + public void before() { + System.getProperties().setProperty(LinkisMainHelper.SERVER_NAME_KEY(), "linkis-et-monitor"); + System.getProperties() + .setProperty("log4j.configurationFile", "src/test/resources/log4j2-console.xml"); + // System.getProperties().setProperty("wds.linkis.server.conf", + // "linkis-et-monitor.properties"); + } + + // @org.junit.Test + public void doSendAlert() throws Exception { + Set ways = new HashSet<>(); + ways.add(ImsAlertWay.WeChat()); + ways.add(ImsAlertWay.Email()); + + Set receivers = new HashSet<>(); + receivers.add("shangda, johnnwang"); + ImsAlertDesc desc = + new ImsAlertDesc( + "5435", + "linkis_alert_test", + "linkis_alert", + "this is a test for linkis", + ImsAlertLevel.MINOR(), + "10.127.0.0.1", + 0, + ways, + receivers, + 3, + 12); + + System.out.println(desc); + String url = "http://172.21.0.130:10812/ims_data_access/send_alarm_by_json.do"; + + PooledImsAlertSender sender = new PooledImsAlertSender("5136", url, receivers); + sender.doSendAlert(desc); + } +} diff --git a/linkis-extensions/linkis-et-monitor/src/test/java/org/apache/linkis/monitor/scan/utils/alert/PooledImsAlertUtilsTest.java b/linkis-extensions/linkis-et-monitor/src/test/java/org/apache/linkis/monitor/scan/utils/alert/PooledImsAlertUtilsTest.java new file mode 100644 index 0000000000..8ca5ff1a98 --- /dev/null +++ b/linkis-extensions/linkis-et-monitor/src/test/java/org/apache/linkis/monitor/scan/utils/alert/PooledImsAlertUtilsTest.java @@ -0,0 +1,49 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.monitor.scan.utils.alert; + +import org.apache.linkis.monitor.scan.constants.Constants; +import org.apache.linkis.monitor.scan.utils.alert.ims.JobHistoryScanImsAlertPropFileParserUtils; +import org.apache.linkis.monitor.scan.utils.alert.ims.PooledImsAlertUtils; +import org.apache.linkis.server.utils.LinkisMainHelper; + +import java.util.Map; + +public class PooledImsAlertUtilsTest { + // @Before + public void before() { + System.getProperties().setProperty(LinkisMainHelper.SERVER_NAME_KEY(), "linkis-et-monitor"); + System.getProperties() + .setProperty("log4j.configurationFile", "src/test/resources/log4j2-console.xml"); + // System.getProperties().setProperty("wds.linkis.server.conf", + // "linkis-et-monitor.properties"); + } + + // @Test + public void addAlert() throws Exception { + PooledImsAlertUtils.addAlert("1st test"); + Map alerts = + JobHistoryScanImsAlertPropFileParserUtils.getAlerts(Constants.SCAN_PREFIX_ERRORCODE()); + for (Map.Entry kv : alerts.entrySet()) { + System.out.println(kv.getKey() + ": " + kv.getValue().toString()); + PooledImsAlertUtils.addAlert(kv.getValue()); + } + Thread.sleep(2000l); + PooledImsAlertUtils.shutDown(true, -1); + } +} diff --git a/linkis-extensions/linkis-et-monitor/src/test/resources/linkis-et-jobhistory-scan-ims.properties b/linkis-extensions/linkis-et-monitor/src/test/resources/linkis-et-jobhistory-scan-ims.properties new file mode 100644 index 0000000000..f08d130651 --- /dev/null +++ b/linkis-extensions/linkis-et-monitor/src/test/resources/linkis-et-jobhistory-scan-ims.properties @@ -0,0 +1,6 @@ +jobhistory.errorcode.11001={"alert_title":"linkis_alert_test","alert_info":"this is a test for linkis errorcode","alert_way":"3,2","alert_reciver":"shangda","alert_level":"2","alert_obj":"linkis_alert","can_recover":"0"} +jobhistory.errorcode.11002={"alert_title":"linkis_alert_test","alert_info":"this is a test for linkis errorcode","alert_way":"3,2","alert_reciver":"shangda","alert_level":"2","alert_obj":"linkis_alert","can_recover":"0"} +jobhistory.errorcode.12011={"alert_title":"linkis_alert_test","alert_info":"this is a test for linkis errorcode","alert_way":"3,2","alert_reciver":"shangda","alert_level":"2","alert_obj":"linkis_alert","can_recover":"0"} +jobhistory.errorcode.43003={"alert_title":"linkis_alert_test","alert_info":"this is a test for linkis errorcode","alert_way":"3,2","alert_reciver":"shangda","alert_level":"2","alert_obj":"linkis_alert","can_recover":"0"} +jobhistory.errorcode.42011={"alert_title":"linkis_alert_test","alert_info":"this is a test for linkis errorcode","alert_way":"3,2","alert_reciver":"shangda","alert_level":"2","alert_obj":"linkis_alert","can_recover":"0"} +jobhistory.unfinished.time.exceed.sec.300={"alert_title":"linkis_alert_test_job_time","alert_info":"this is a test for linkis 5 job time exceed","alert_way":"3,2","alert_reciver":"shangda","alert_level":"2","alert_obj":"linkis_alert","can_recover":"0"} \ No newline at end of file diff --git a/linkis-extensions/linkis-et-monitor/src/test/resources/linkis-et-jobhistory-scan.properties b/linkis-extensions/linkis-et-monitor/src/test/resources/linkis-et-jobhistory-scan.properties new file mode 100644 index 0000000000..0eca42bd5d --- /dev/null +++ b/linkis-extensions/linkis-et-monitor/src/test/resources/linkis-et-jobhistory-scan.properties @@ -0,0 +1,14 @@ +wds.linkis.server.mybatis.mapperLocations=classpath*:org/apache/linkis/jobhistory/scan/app/jobhistory/dao/impl/*.xml +wds.linkis.server.mybatis.typeAliasesPackage=org.apache.linkis.monitor.scan.app.jobhistory.entity +wds.linkis.server.mybatis.BasePackage=org.apache.linkis.monitor.scan.app.jobhistory.dao +wds.linkis.mysql.is.encrypt=false +#wds.linkis.server.mybatis.datasource.url=jdbc:mysql://10.108.161.105:15304/bdp_easy_ide?characterEncoding=UTF-8 +#wds.linkis.server.mybatis.datasource.username=bdpeasyide +#wds.linkis.server.mybatis.datasource.password=bdpeasyide@bdpsit +wds.linkis.server.mybatis.datasource.url=jdbc:mysql://10.107.108.111:3306/vsbi_gz_bdap_sit_01?characterEncoding=UTF-8 +wds.linkis.server.mybatis.datasource.username=bdp_vsbi +wds.linkis.server.mybatis.datasource.password=bdpVsbi@2019 +wds.linkis.alert.url=http://172.21.0.130:10812/ims_data_access/send_alarm_by_json.do +wds.linkis.alert.receiver.default=johnnwang +wds.linkis.errorcode.scanner.interval.seconds=7200 +wds.linkis.errorcode.scanner.max.interval.seconds=3600 \ No newline at end of file diff --git a/linkis-extensions/linkis-et-monitor/src/test/resources/log4j2-console.xml b/linkis-extensions/linkis-et-monitor/src/test/resources/log4j2-console.xml new file mode 100644 index 0000000000..49eabc51e6 --- /dev/null +++ b/linkis-extensions/linkis-et-monitor/src/test/resources/log4j2-console.xml @@ -0,0 +1,46 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/linkis-extensions/pom.xml b/linkis-extensions/pom.xml index 661b6be0bf..7233141565 100644 --- a/linkis-extensions/pom.xml +++ b/linkis-extensions/pom.xml @@ -28,6 +28,7 @@ pom linkis-io-file-client + linkis-et-monitor From e88d5b5b70bd90384d5a3d60ba0d0090cafa7a2d Mon Sep 17 00:00:00 2001 From: peacewong Date: Tue, 5 Sep 2023 21:25:42 +0800 Subject: [PATCH 2/6] code optimize --- linkis-extensions/linkis-et-monitor/pom.xml | 10 +- .../src/main/assembly/distribution.xml | 6 +- .../app/LinkisDirtyDataCleanApplication.java | 111 ----- .../app/LinkisJobHistoryScanApplication.java | 172 -------- .../app/instance/dao/InstanceLabelDao.java | 1 - .../instance/entity/InsPersistenceLabel.java | 16 + .../app/jobhistory/dao/JobHistoryMapper.java | 1 - .../app/monitor/config/MonitorConfig.java | 33 +- .../app/monitor/entity/ChatbotEntity.java | 114 ++--- .../scan/app/monitor/scheduled/BmlClear.java | 1 - .../app/monitor/scheduled/EcRecordClear.java | 4 +- .../monitor/scheduled/JobHistoryClear.java | 4 +- .../monitor/scheduled/JobHistoryMonitor.java | 395 +++++++++--------- .../monitor/scheduled/ResourceMonitor.java | 73 ++-- .../app/monitor/scheduled/TaskLogClear.java | 4 +- .../app/monitor/scheduled/TaskMonitor.java | 78 ++-- .../monitor/scheduled/UserModeMonitor.java | 212 +++++----- .../app/monitor/scheduled/ValidatorClear.java | 34 +- .../scan/app/monitor/until/HttpsUntils.java | 13 +- ...kisJobHistoryScanSpringConfiguration.scala | 4 - .../scan/app/factory/MapperFactory.scala | 19 +- .../jobhistory/JobHistoryDataFetcher.scala | 14 +- .../errorcode/JobHistoryErrCodeHitEvent.scala | 1 - .../errorcode/JobHistoryErrCodeRule.scala | 20 +- .../JobHistoryErrorCodeAlertSender.scala | 5 +- .../jobtime/JobTimeExceedAlertSender.scala | 35 +- .../jobtime/JobTimeExceedHitEvent.scala | 1 - .../jobtime/JobTimeExceedRule.scala | 15 +- .../labels/JobHistoryLabelsAlertSender.scala | 4 +- .../labels/JobHistoryLabelsRule.scala | 20 +- .../runtime/CommonJobRunTimeRule.scala | 26 +- .../runtime/CommonRunTimeAlertSender.scala | 38 +- .../JobHistoryRunTimeAlertSender.scala | 19 +- .../runtime/JobHistoryRunTimeRule.scala | 33 +- .../scan/client/MonitorHTTPClient.scala | 6 +- .../monitor/scan/constants/Constants.scala | 68 +-- .../scan/constants/ScanOperatorEnum.scala | 1 - .../linkis/monitor/scan/core/ob/Event.scala | 1 - .../monitor/scan/core/ob/Observer.scala | 1 - .../scan/core/pac/AbstractDataFetcher.scala | 1 - .../scan/core/pac/AbstractScanRule.scala | 1 - .../scan/core/pac/BaseScannedData.scala | 1 - .../monitor/scan/core/pac/ScanBuffer.scala | 1 - .../monitor/scan/core/pac/ScannedData.scala | 1 - .../scan/core/scanner/AbstractScanner.scala | 24 +- .../scan/core/scanner/AnomalyScanner.scala | 8 +- .../scan/core/scanner/DefaultScanner.scala | 1 - .../scan/request/EntranceTaskAction.scala | 3 +- .../monitor/scan/utils/alert/AlertDesc.scala | 1 - .../scan/utils/alert/AlertSender.scala | 1 - .../scan/utils/alert/PooledAlertSender.scala | 17 +- .../scan/utils/alert/ims/ImsAlertDesc.scala | 3 +- .../scan/utils/alert/ims/ImsAlertLevel.scala | 1 - .../alert/ims/ImsAlertPropFileData.scala} | 30 +- .../scan/utils/alert/ims/ImsAlertWay.scala | 3 +- .../scan/utils/alert/ims/ImsRequest.scala | 1 - ...storyScanImsAlertPropFileParserUtils.scala | 132 ------ .../utils/alert/ims/MonitorAlertUtils.scala | 171 +++----- .../alert/ims/PooledImsAlertSender.scala | 14 +- .../utils/alert/ims/PooledImsAlertUtils.scala | 11 +- .../utils/alert/ims/UserLabelAlertUtils.scala | 95 +---- .../monitor/scan/utils/log/LogUtils.scala | 1 - ...ryScanImsAlertPropFileParserUtilsTest.java | 48 --- .../utils/alert/PooledImsAlertUtilsTest.java | 4 +- 64 files changed, 814 insertions(+), 1373 deletions(-) delete mode 100644 linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/LinkisDirtyDataCleanApplication.java delete mode 100644 linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/LinkisJobHistoryScanApplication.java rename linkis-extensions/linkis-et-monitor/src/{test/java/org/apache/linkis/monitor/scan/app/LinkisJobHistoryScanApplicationTest.java => main/scala/org/apache/linkis/monitor/scan/utils/alert/ims/ImsAlertPropFileData.scala} (52%) delete mode 100644 linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/utils/alert/ims/JobHistoryScanImsAlertPropFileParserUtils.scala delete mode 100644 linkis-extensions/linkis-et-monitor/src/test/java/org/apache/linkis/monitor/scan/app/JobHistoryScanImsAlertPropFileParserUtilsTest.java diff --git a/linkis-extensions/linkis-et-monitor/pom.xml b/linkis-extensions/linkis-et-monitor/pom.xml index 5e62fa60b5..6972c9e1a4 100644 --- a/linkis-extensions/linkis-et-monitor/pom.xml +++ b/linkis-extensions/linkis-et-monitor/pom.xml @@ -87,11 +87,11 @@ ${project.version} provided - - org.apache.linkis - linkis-rpc - ${project.version} - + + org.apache.linkis + linkis-rpc + ${project.version} + diff --git a/linkis-extensions/linkis-et-monitor/src/main/assembly/distribution.xml b/linkis-extensions/linkis-et-monitor/src/main/assembly/distribution.xml index 8cc9e7ea7b..647b84c833 100644 --- a/linkis-extensions/linkis-et-monitor/src/main/assembly/distribution.xml +++ b/linkis-extensions/linkis-et-monitor/src/main/assembly/distribution.xml @@ -16,10 +16,8 @@ ~ limitations under the License. --> - + linkis-et-monitor dir diff --git a/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/LinkisDirtyDataCleanApplication.java b/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/LinkisDirtyDataCleanApplication.java deleted file mode 100644 index 3530ef5139..0000000000 --- a/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/LinkisDirtyDataCleanApplication.java +++ /dev/null @@ -1,111 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.monitor.scan.app; - -import org.apache.linkis.monitor.scan.app.dirtydata.entrance.EntranceDirtyDataHandler; -import org.apache.linkis.monitor.scan.utils.log.LogUtils; -import org.apache.linkis.server.utils.LinkisMainHelper; - -import org.apache.commons.lang3.StringUtils; - -import org.springframework.context.annotation.AnnotationConfigApplicationContext; -import org.springframework.context.support.AbstractApplicationContext; - -import org.slf4j.Logger; - -public class LinkisDirtyDataCleanApplication { - private static final Logger logger = LogUtils.stdOutLogger(); - - /** @param args: args[0]: host args[1] port */ - public static void main(String[] args) throws ReflectiveOperationException { - if (args.length == 0 - || StringUtils.equalsIgnoreCase(args[0], "help") - || StringUtils.equalsIgnoreCase(args[0], "--help")) { - LogUtils.stdOutLogger() - .info( - "[help-message]this app cleans entrance dirty-data. args[0]: command-type (entrance/help/...) args[1]: entrance-hostname(not null), args[2]: entrance-port(can be null)"); - return; - } - String serviceName = System.getProperty(LinkisMainHelper.SERVER_NAME_KEY()); - LinkisMainHelper.formatPropertyFiles(serviceName); - - if (StringUtils.equalsIgnoreCase(args[0], "entrance")) { - AbstractApplicationContext context = - new AnnotationConfigApplicationContext(LinkisJobHistoryScanSpringConfiguration.class); - - String host = ""; - String port = ""; - if (args.length > 1) { - host = args[1]; - } - if (args.length > 2) { - port = args[2]; - } - if (args.length > 3) { - printIllegalInput("wrong number of arguments"); - return; - } - try { - removeDirtyEurekaInstance(host, port); - } catch (Exception e) { - LogUtils.stdOutLogger().error("Failed to remove dirty eureka-instance", e); - } - try { - removeDbDirtyData(host, port); - } catch (Exception e) { - LogUtils.stdOutLogger().error("Failed to remove dirty db-data", e); - } - - context.close(); - } else { - LogUtils.stdOutLogger().error("Upsupported command type: " + args[0]); - } - } - - private static void printIllegalInput(String msg) { - LogUtils.stdOutLogger().error("illegal input: " + msg); - LogUtils.stdOutLogger() - .info( - "[help-message] this app cleans entrance dirty-data. args[0]: entrance-hostname, args[1]: entrance-port"); - return; - } - - private static void removeDirtyEurekaInstance(String host, String port) { - if (StringUtils.isBlank(host)) { - printIllegalInput("host cannot be blank"); - return; - } - if (StringUtils.isBlank(port)) { - EntranceDirtyDataHandler.handleEurekaDirtyData(host); - } else { - EntranceDirtyDataHandler.handleEurekaDirtyData(host, port); - } - } - - private static void removeDbDirtyData(String host, String port) { - if (StringUtils.isBlank(host)) { - printIllegalInput("host cannot be blank"); - return; - } - if (StringUtils.isBlank(port)) { - EntranceDirtyDataHandler.handleDbDirtData(host); - } else { - EntranceDirtyDataHandler.handleDbDirtData(host, port); - } - } -} diff --git a/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/LinkisJobHistoryScanApplication.java b/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/LinkisJobHistoryScanApplication.java deleted file mode 100644 index 8b4366805e..0000000000 --- a/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/LinkisJobHistoryScanApplication.java +++ /dev/null @@ -1,172 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.monitor.scan.app; - -import org.apache.linkis.monitor.scan.app.factory.MapperFactory; -import org.apache.linkis.monitor.scan.app.jobhistory.JobHistoryDataFetcher; -import org.apache.linkis.monitor.scan.app.jobhistory.errorcode.JobHistoryErrCodeRule; -import org.apache.linkis.monitor.scan.app.jobhistory.errorcode.JobHistoryErrorCodeAlertSender; -import org.apache.linkis.monitor.scan.app.jobhistory.jobtime.JobTimeExceedAlertSender; -import org.apache.linkis.monitor.scan.app.jobhistory.jobtime.JobTimeExceedRule; -import org.apache.linkis.monitor.scan.constants.Constants; -import org.apache.linkis.monitor.scan.core.pac.DataFetcher; -import org.apache.linkis.monitor.scan.core.scanner.AnomalyScanner; -import org.apache.linkis.monitor.scan.core.scanner.DefaultScanner; -import org.apache.linkis.monitor.scan.utils.alert.AlertDesc; -import org.apache.linkis.monitor.scan.utils.alert.ims.ImsAlertDesc; -import org.apache.linkis.monitor.scan.utils.alert.ims.JobHistoryScanImsAlertPropFileParserUtils; -import org.apache.linkis.monitor.scan.utils.log.LogUtils; -import org.apache.linkis.server.utils.LinkisMainHelper; - -import org.apache.commons.lang3.exception.ExceptionUtils; - -import org.springframework.context.annotation.AnnotationConfigApplicationContext; -import org.springframework.context.support.AbstractApplicationContext; - -import java.text.ParseException; -import java.text.SimpleDateFormat; -import java.util.ArrayList; -import java.util.Date; -import java.util.List; -import java.util.Map; - -import org.slf4j.Logger; - -public class LinkisJobHistoryScanApplication { - private static final Logger logger = LogUtils.stdOutLogger(); - - /** @param args: args[0]: startTime, args[1] endTime */ - public static void main(String[] args) throws ReflectiveOperationException { - - String serviceName = System.getProperty(LinkisMainHelper.SERVER_NAME_KEY()); - LinkisMainHelper.formatPropertyFiles(serviceName); - - long intervalMs = Constants.SCAN_INTERVALS_SECONDS() * 1000; - long maxIntervalMs = Constants.MAX_INTERVALS_SECONDS() * 1000; - long endTime = System.currentTimeMillis(); - long startTime = endTime - intervalMs; - /** parse input into timestamp */ - if (args != null && args.length == 2) { - SimpleDateFormat format = new SimpleDateFormat("yyyyMMddHH"); - Long tmpStart; - Long tmpEnd; - try { - tmpStart = format.parse(args[0]).getTime(); - tmpEnd = format.parse(args[1]).getTime(); - } catch (ParseException e) { - logger.info( - "Failed to parse input into startTime and endTime." + ExceptionUtils.getMessage(e)); - return; - } - if (endTime <= startTime) { - logger.info("[error] startTime larger than endTime"); - return; - } - if (tmpStart != null && tmpEnd != null) { - startTime = tmpStart; - endTime = tmpEnd; - } - } - long realIntervals = endTime - startTime < maxIntervalMs ? endTime - startTime : maxIntervalMs; - - runApp(startTime, endTime, realIntervals, maxIntervalMs); - } - - private static void runApp(long startTime, long endTime, long realIntervals, long maxIntervalMs) { - - AbstractApplicationContext context = - new AnnotationConfigApplicationContext(LinkisJobHistoryScanSpringConfiguration.class); - - AnomalyScanner scanner = new DefaultScanner(); - boolean shouldStart = false; - - List fetchers = generateFetchers(startTime, endTime, maxIntervalMs); - if (fetchers == null) { - logger.warn("generated 0 dataFetchers, plz check input"); - return; - } - - Map errorCodeAlerts = - JobHistoryScanImsAlertPropFileParserUtils.getAlerts(Constants.SCAN_PREFIX_ERRORCODE()); - if (errorCodeAlerts == null || errorCodeAlerts.size() == 0) { - logger.info("[INFO] Loaded 0 errorcode alert from alert-rule properties file."); - } else { - logger.info( - "[INFO] Loaded {} errorcode alert from alert-rules properties file.", - errorCodeAlerts.size()); - shouldStart = true; - addIntervalToImsAlerts(errorCodeAlerts, realIntervals); - JobHistoryErrCodeRule jobHistoryErrCodeRule = - new JobHistoryErrCodeRule( - errorCodeAlerts.keySet(), new JobHistoryErrorCodeAlertSender(errorCodeAlerts)); - scanner.addScanRule(jobHistoryErrCodeRule); - } - - Map jobTimeAlerts = - JobHistoryScanImsAlertPropFileParserUtils.getAlerts( - Constants.SCAN_PREFIX_UNFINISHED_JOBTIME_EXCEED_SEC()); - if (jobTimeAlerts == null || jobTimeAlerts.size() == 0) { - logger.info("[INFO] Loaded 0 alerts jobtime alert-rule from alert properties file."); - } else { - logger.info( - "[INFO] Loaded {} alerts jobtime alert-rules from alert properties file.", - jobTimeAlerts.size()); - shouldStart = true; - addIntervalToImsAlerts(jobTimeAlerts, realIntervals); - JobTimeExceedRule jobTimeExceedRule = - new JobTimeExceedRule( - jobTimeAlerts.keySet(), new JobTimeExceedAlertSender(jobTimeAlerts)); - scanner.addScanRule(jobTimeExceedRule); - } - if (shouldStart) { - scanner.addDataFetchers(fetchers); - scanner.run(); - scanner.shutdown(); // wait all alert to be send - } - context.close(); - } - - private static List generateFetchers( - long startTime, long endTime, long maxIntervalMs) { - List ret = new ArrayList<>(); - long pe = endTime; - long ps; - while (pe > startTime) { - ps = pe - maxIntervalMs > startTime ? pe - maxIntervalMs : startTime; - String[] fetcherArgs = new String[] {String.valueOf(ps), String.valueOf(pe)}; - ret.add(new JobHistoryDataFetcher(fetcherArgs, MapperFactory.getJobHistoryMapper())); - logger.info( - "Generated dataFetcher for startTime: " - + new Date(ps).toString() - + ". EndTime: " - + new Date(pe).toString()); - pe = pe - maxIntervalMs; - } - return ret; - } - - private static void addIntervalToImsAlerts(Map alerts, long realIntervals) { - for (AlertDesc alert : alerts.values()) { - if (!(alert instanceof ImsAlertDesc)) { - logger.info("[warn] ignore wrong alert" + alert); - } else { - ((ImsAlertDesc) alert).hitIntervalMs_$eq(realIntervals); - } - } - } -} diff --git a/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/instance/dao/InstanceLabelDao.java b/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/instance/dao/InstanceLabelDao.java index 8f73f0349a..5fa3b7a5c0 100644 --- a/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/instance/dao/InstanceLabelDao.java +++ b/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/instance/dao/InstanceLabelDao.java @@ -21,7 +21,6 @@ import java.util.List; - public interface InstanceLabelDao { /** * Remove label diff --git a/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/instance/entity/InsPersistenceLabel.java b/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/instance/entity/InsPersistenceLabel.java index e828328b43..36fb21c13b 100644 --- a/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/instance/entity/InsPersistenceLabel.java +++ b/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/instance/entity/InsPersistenceLabel.java @@ -79,4 +79,20 @@ public Date getCreateTime() { public void setCreateTime(Date createTime) { this.createTime = createTime; } + + public String getLabelKey() { + return labelKey; + } + + public void setLabelKey(String labelKey) { + this.labelKey = labelKey; + } + + public String getFeature() { + return feature; + } + + public void setFeature(String feature) { + this.feature = feature; + } } diff --git a/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/jobhistory/dao/JobHistoryMapper.java b/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/jobhistory/dao/JobHistoryMapper.java index b3774860af..bfdc640652 100644 --- a/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/jobhistory/dao/JobHistoryMapper.java +++ b/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/jobhistory/dao/JobHistoryMapper.java @@ -24,7 +24,6 @@ import java.util.Date; import java.util.List; - public interface JobHistoryMapper { List selectJobHistory(JobHistory jobReq); diff --git a/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/monitor/config/MonitorConfig.java b/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/monitor/config/MonitorConfig.java index fab9a5cbe7..32b47c289c 100644 --- a/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/monitor/config/MonitorConfig.java +++ b/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/monitor/config/MonitorConfig.java @@ -29,9 +29,9 @@ public class MonitorConfig { public static final CommonVars USER_MODE_TIMEOUT = CommonVars.apply("linkis.monitor.user.timeOut", 300); public static final CommonVars USER_MODE_AUTHTOKEN = - CommonVars.apply("linkis.monitor.user.authToken","VALIDATOR-AUTH"); + CommonVars.apply("linkis.monitor.user.authToken", "VALIDATOR-AUTH"); public static final CommonVars USER_MODE_ENGINE = - CommonVars.apply("linkis.monitor.user.enginelist","[]"); + CommonVars.apply("linkis.monitor.user.enginelist", "[]"); public static final CommonVars ECM_TASK_MAJOR = CommonVars.apply("linkis.monitor.ecmResourceTask.major", 0.03); @@ -43,7 +43,7 @@ public class MonitorConfig { CommonVars.apply("linkis.monitor.metrics.userAuthKey"); public static final CommonVars JOB_HISTORY_TIME_EXCEED = - CommonVars.apply("linkis.monitor.jobhistory.id.timeExceed",0L); + CommonVars.apply("linkis.monitor.jobhistory.id.timeExceed", 0L); public static final CommonVars ENTRANCE_TASK_USERTOTAL = CommonVars.apply("linkis.monitor.entranceTask.userTotalTask", 1000); @@ -52,7 +52,7 @@ public class MonitorConfig { public static final CommonVars ENTRANCE_TASK_TOTAL_MINOR = CommonVars.apply("linkis.monitor.entranceTask.linkisTotalTaskMinor", 10000); public static final CommonVars ENTRANCE_TASK_USERLIST = - CommonVars.apply("linkis.monitor.entranceTask.userlist","[]"); + CommonVars.apply("linkis.monitor.entranceTask.userlist", "[]"); public static final CommonVars SCHEDULED_CONFIG_NUM = CommonVars.apply("linkis.monitor.scheduled.pool.cores.num", 10); @@ -61,11 +61,24 @@ public class MonitorConfig { CommonVars.apply("linkis.monitor.shell.time.out.minute", 30); public static final CommonVars USER_MODE_INTERFACE_TIMEOUT = - CommonVars.apply("linkis.monitor.user.mode.time.out", 30*1000); + CommonVars.apply("linkis.monitor.user.mode.time.out", 30 * 1000); - public static final CommonVars CHATBOT_KEY_ID = CommonVars.apply("linkis.monitor.chatbot.key.id","23e6afad1b78a0c5eed67e4d24de7063"); - public static final CommonVars CHATBOT_TYPE = CommonVars.apply("linkis.monitor.chatbot.type","text"); - public static final CommonVars CHATBOT_SERVICE_NAME= CommonVars.apply("linkis.monitor.chatbot.serviceName","大数据生产助手(BDP_PRD)"); - public static final CommonVars CHATBOT_URL= CommonVars.apply("linkis.monitor.chatbot.url","http://172.21.3.43:1377/pros-chatbot/yuanfang/sendEMsg"); - public static final CommonVars SOLUTION_URL = CommonVars.apply("linkis.monitor.jobhistory.solution.url", "http://kn.dss.weoa.com/linkis/qa"); + public static final CommonVars CHATBOT_KEY_ID = + CommonVars.apply("linkis.monitor.chatbot.key.id", "23e6afad1b78a0c5eed67e4d24de7063"); + public static final CommonVars CHATBOT_TYPE = + CommonVars.apply("linkis.monitor.chatbot.type", "text"); + public static final CommonVars CHATBOT_SERVICE_NAME = + CommonVars.apply("linkis.monitor.chatbot.serviceName", ""); + public static final CommonVars CHATBOT_URL = + CommonVars.apply("linkis.monitor.chatbot.url", ""); + public static final CommonVars SOLUTION_URL = + CommonVars.apply( + "linkis.monitor.jobhistory.solution.url", + "https://linkis.apache.org/docs/latest/tuning-and-troubleshooting/error-guide/error-code"); + + public static final CommonVars TASK_RUNTIME_TIMEOUT_DESC = + CommonVars.apply( + "linkis.monitor.jobhistory.task.timeout.desc", + "[Linkis任务信息]您好，您在Linkis/DSS提交的任务(任务ID:{0})，已经运行超过{1}h，" + + "请关注是否任务正常，如果不正常您可以到Linkis/DSS管理台进行任务的kill，集群信息为BDAP({2})。详细解决方案见Q47：{3} "); } diff --git a/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/monitor/entity/ChatbotEntity.java b/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/monitor/entity/ChatbotEntity.java index 3cf288adbb..4f912b9e82 100644 --- a/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/monitor/entity/ChatbotEntity.java +++ b/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/monitor/entity/ChatbotEntity.java @@ -20,68 +20,78 @@ import org.apache.linkis.monitor.scan.app.monitor.config.MonitorConfig; public class ChatbotEntity { - private String keyid; - private String content; - private String type; - private String userName; - private String serviceName; + private String keyid; + private String content; + private String type; + private String userName; + private String serviceName; - public ChatbotEntity(String content, String userName) { - this.keyid = MonitorConfig.CHATBOT_KEY_ID.getValue(); - this.content = content; - this.type = MonitorConfig.CHATBOT_TYPE.getValue(); - this.userName = userName; - this.serviceName = MonitorConfig.CHATBOT_SERVICE_NAME.getValue(); - } + public ChatbotEntity(String content, String userName) { + this.keyid = MonitorConfig.CHATBOT_KEY_ID.getValue(); + this.content = content; + this.type = MonitorConfig.CHATBOT_TYPE.getValue(); + this.userName = userName; + this.serviceName = MonitorConfig.CHATBOT_SERVICE_NAME.getValue(); + } - public String getKeyid() { - return keyid; - } + public String getKeyid() { + return keyid; + } - public void setKeyid(String keyid) { - this.keyid = keyid; - } + public void setKeyid(String keyid) { + this.keyid = keyid; + } - public String getContent() { - return content; - } + public String getContent() { + return content; + } - public void setContent(String content) { - this.content = content; - } + public void setContent(String content) { + this.content = content; + } - public String getType() { - return type; - } + public String getType() { + return type; + } - public void setType(String type) { - this.type = type; - } + public void setType(String type) { + this.type = type; + } - public String getUserName() { - return userName; - } + public String getUserName() { + return userName; + } - public void setUserName(String userName) { - this.userName = userName; - } + public void setUserName(String userName) { + this.userName = userName; + } - public String getServiceName() { - return serviceName; - } + public String getServiceName() { + return serviceName; + } - public void setServiceName(String serviceNameuserName) { - this.serviceName = serviceNameuserName; - } + public void setServiceName(String serviceNameuserName) { + this.serviceName = serviceNameuserName; + } - @Override - public String toString() { - return "ChatbotEntity{" + - "keyid='" + keyid + '\'' + - ", content='" + content + '\'' + - ", type='" + type + '\'' + - ", userName='" + userName + '\'' + - ", serviceName='" + serviceName + '\'' + - '}'; - } + @Override + public String toString() { + return "ChatbotEntity{" + + "keyid='" + + keyid + + '\'' + + ", content='" + + content + + '\'' + + ", type='" + + type + + '\'' + + ", userName='" + + userName + + '\'' + + ", serviceName='" + + serviceName + + '\'' + + '}'; + } } diff --git a/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/monitor/scheduled/BmlClear.java b/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/monitor/scheduled/BmlClear.java index 75d415ac0d..11ee06226f 100644 --- a/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/monitor/scheduled/BmlClear.java +++ b/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/monitor/scheduled/BmlClear.java @@ -29,7 +29,6 @@ import org.slf4j.Logger; - @Component @PropertySource(value = "classpath:linkis-et-monitor.properties", encoding = "UTF-8") public class BmlClear { diff --git a/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/monitor/scheduled/EcRecordClear.java b/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/monitor/scheduled/EcRecordClear.java index dced6117b1..ac9cb4ea61 100644 --- a/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/monitor/scheduled/EcRecordClear.java +++ b/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/monitor/scheduled/EcRecordClear.java @@ -30,9 +30,7 @@ import org.slf4j.Logger; -/*** - * Task: clean up linkis_cg_ec_resource_info_record data - */ +/** * Task: clean up linkis_cg_ec_resource_info_record data */ @Component @PropertySource(value = "classpath:linkis-et-monitor.properties", encoding = "UTF-8") public class EcRecordClear { diff --git a/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/monitor/scheduled/JobHistoryClear.java b/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/monitor/scheduled/JobHistoryClear.java index 4880a600cc..ae14770420 100644 --- a/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/monitor/scheduled/JobHistoryClear.java +++ b/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/monitor/scheduled/JobHistoryClear.java @@ -30,9 +30,7 @@ import org.slf4j.Logger; -/*** - * Task: clean up linkis_ps_job_history_group_history data - */ +/** * Task: clean up linkis_ps_job_history_group_history data */ @Component @PropertySource(value = "classpath:linkis-et-monitor.properties", encoding = "UTF-8") public class JobHistoryClear { diff --git a/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/monitor/scheduled/JobHistoryMonitor.java b/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/monitor/scheduled/JobHistoryMonitor.java index 01873a988b..fbca945a8d 100644 --- a/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/monitor/scheduled/JobHistoryMonitor.java +++ b/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/monitor/scheduled/JobHistoryMonitor.java @@ -25,8 +25,8 @@ import org.apache.linkis.monitor.scan.app.jobhistory.jobtime.JobTimeExceedRule; import org.apache.linkis.monitor.scan.app.jobhistory.labels.JobHistoryLabelsAlertSender; import org.apache.linkis.monitor.scan.app.jobhistory.labels.JobHistoryLabelsRule; -import org.apache.linkis.monitor.scan.app.jobhistory.runtime.CommonRunTimeAlertSender; import org.apache.linkis.monitor.scan.app.jobhistory.runtime.CommonJobRunTimeRule; +import org.apache.linkis.monitor.scan.app.jobhistory.runtime.CommonRunTimeAlertSender; import org.apache.linkis.monitor.scan.app.jobhistory.runtime.JobHistoryRunTimeAlertSender; import org.apache.linkis.monitor.scan.app.jobhistory.runtime.JobHistoryRunTimeRule; import org.apache.linkis.monitor.scan.app.monitor.config.MonitorConfig; @@ -37,7 +37,7 @@ import org.apache.linkis.monitor.scan.core.scanner.DefaultScanner; import org.apache.linkis.monitor.scan.utils.alert.AlertDesc; import org.apache.linkis.monitor.scan.utils.alert.ims.ImsAlertDesc; -import org.apache.linkis.monitor.scan.utils.alert.ims.JobHistoryScanImsAlertPropFileParserUtils; +import org.apache.linkis.monitor.scan.utils.alert.ims.MonitorAlertUtils; import org.apache.linkis.monitor.scan.utils.alert.ims.UserLabelAlertUtils; import org.apache.linkis.monitor.scan.utils.log.LogUtils; @@ -49,216 +49,219 @@ import org.slf4j.Logger; -/*** - * jobHistory monitor - * 1.errorCode: Monitor the error code - * 2.userLabel: tenant label monitoring, scan the execution data within the first 20 minutes, and judge the labels field of the data - * 3.jobResultRunTime: Scan the execution data within the first 20 minutes, and judge the completed tasks. If the parm field in the jobhistory contains (task.notification.conditions) and the result of executing the task is (Succeed, Failed, Canceled, Timeout, ALL) any one of them, an alarm is triggered and the result of the job is that it has ended. All three are indispensable - * 4.jobResultRunTimeForDSS: Scan the execution data within the first 20 minutes, scan the tasks that have been marked for notification, if the task has ended, a notification will be initiated - * 5.jobHistoryUnfinishedScan: monitor the status of the execution task, scan the data outside 12 hours and within 24 hours +/** + * * jobHistory monitor 1.errorCode: Monitor the error code 2.userLabel: tenant label monitoring, + * scan the execution data within the first 20 minutes, and judge the labels field of the data + * 3.jobResultRunTime: Scan the execution data within the first 20 minutes, and judge the completed + * tasks. If the parm field in the jobhistory contains (task.notification.conditions) and the result + * of executing the task is (Succeed, Failed, Canceled, Timeout, ALL) any one of them, an alarm is + * triggered and the result of the job is that it has ended. All three are indispensable + * 4.jobResultRunTimeForDSS: Scan the execution data within the first 20 minutes, scan the tasks + * that have been marked for notification, if the task has ended, a notification will be initiated + * 5.jobHistoryUnfinishedScan: monitor the status of the execution task, scan the data outside 12 + * hours and within 24 hours */ @Component @PropertySource(value = "classpath:linkis-et-monitor.properties", encoding = "UTF-8") public class JobHistoryMonitor { - private static final Logger logger = LogUtils.stdOutLogger(); - private static final long backtrackNum = 1000000L; + private static final Logger logger = LogUtils.stdOutLogger(); + private static final long backtrackNum = 1000000L; - /** - * Scan tasks completed within 20 minutes - */ - @Scheduled(cron = "0 0/10 * * * ?") - public void jobHistoryFinishedScan() { - long intervalMs = 20 * 60 * 1000; // 20分钟 - long maxIntervalMs = Constants.MAX_INTERVALS_SECONDS() * 1000; - long endTime = System.currentTimeMillis(); - long startTime = endTime - intervalMs; - long realIntervals = endTime - startTime < maxIntervalMs ? endTime - startTime : maxIntervalMs; - AnomalyScanner scanner = new DefaultScanner(); - boolean shouldStart = false; - long id; - if (null == CacheUtils.cacheBuilder.getIfPresent("jobHistoryId")) { - //20230206：新增获取最大值-100W 作为初始id进行查询，防止慢查询 - long maxId = MapperFactory.getJobHistoryMapper().selectMaxId(); - long beginId = 0L; - if (maxId > backtrackNum) { - beginId = maxId - backtrackNum; - } - id = MapperFactory.getJobHistoryMapper().selectIdByHalfDay(beginId); - CacheUtils.cacheBuilder.put("jobHistoryId", id); - } else { - id = CacheUtils.cacheBuilder.getIfPresent("jobHistoryId"); - } - List fetchers = generateFetchersfortime(startTime, endTime, id, "updated_time"); - if (fetchers == null) { - logger.warn("generated 0 dataFetchers, plz check input"); - return; - } - // errorCode - try { - Map errorCodeAlerts = JobHistoryScanImsAlertPropFileParserUtils.getAlerts(Constants.SCAN_PREFIX_ERRORCODE()); - - if (errorCodeAlerts == null || errorCodeAlerts.size() == 0) { - logger.info("[INFO] Loaded 0 errorcode alert from alert-rule properties file."); - } else { - logger.info( - "[INFO] Loaded {} errorcode alert from alert-rules properties file.", - errorCodeAlerts.size()); - shouldStart = true; - addIntervalToImsAlerts(errorCodeAlerts, realIntervals); - JobHistoryErrCodeRule jobHistoryErrCodeRule = - new JobHistoryErrCodeRule( - errorCodeAlerts.keySet(), new JobHistoryErrorCodeAlertSender(errorCodeAlerts)); - scanner.addScanRule(jobHistoryErrCodeRule); - } - } catch (Exception e) { - logger.warn("Jobhistory Monitor ErrorCode Faily: "+ e.getMessage()); - } - // userLabel - try { - Map userLabelAlerts = - UserLabelAlertUtils.getAlerts(Constants.USER_LABEL_MONITOR(), ""); - if (userLabelAlerts == null || userLabelAlerts.size() == 0) { - logger.info("[INFO] Loaded 0 alerts userLabel alert-rule from alert properties file."); - } else { - logger.info( - "[INFO] Loaded {} alerts userLabel alert-rules from alert properties file.", - userLabelAlerts.size()); - shouldStart = true; - JobHistoryLabelsRule jobHistoryLabelsRule = - new JobHistoryLabelsRule(new JobHistoryLabelsAlertSender()); - scanner.addScanRule(jobHistoryLabelsRule); - } - } catch (Exception e) { - logger.warn("Jobhistory Monitor UserLabel Faily: "+ e.getMessage()); - } - // jobResultRunTime - try { - Map jobResultAlerts = - JobHistoryScanImsAlertPropFileParserUtils.getAlerts(Constants.SCAN_PREFIX_ERRORCODE()); - if (jobResultAlerts == null || jobResultAlerts.size() == 0) { - logger.info("[INFO] Loaded 0 jobResult alert from alert-rule properties file."); - } else { - logger.info( - "[INFO] Loaded {} alerts jobResult alert-rules from alert properties file.", - jobResultAlerts.size()); - shouldStart = true; - JobHistoryRunTimeRule jobHistoryRunTimeRule = - new JobHistoryRunTimeRule(new JobHistoryRunTimeAlertSender()); - scanner.addScanRule(jobHistoryRunTimeRule); - } - } catch (Exception e) { - logger.warn("Jobhistory Monitor JobResultRunTime Faily: "+ e.getMessage()); - } - // jobResultRunTimeForDSS - try { - Map dssJobResultAlerts = - JobHistoryScanImsAlertPropFileParserUtils.getAlerts(Constants.SCAN_PREFIX_ERRORCODE()); - if (dssJobResultAlerts == null || dssJobResultAlerts.size() == 0) { - logger.info("[INFO] Loaded 0 jobResult alert from alert-rule properties file."); - } else { - logger.info( - "[INFO] Loaded {} alerts jobResult alert-rules from alert properties file.", - dssJobResultAlerts.size()); - shouldStart = true; + /** Scan tasks completed within 20 minutes */ + @Scheduled(cron = "0 0/10 * * * ?") + public void jobHistoryFinishedScan() { + long intervalMs = 20 * 60 * 1000; // 20分钟 + long maxIntervalMs = Constants.ERRORCODE_MAX_INTERVALS_SECONDS() * 1000; + long endTime = System.currentTimeMillis(); + long startTime = endTime - intervalMs; + long realIntervals = endTime - startTime < maxIntervalMs ? endTime - startTime : maxIntervalMs; + AnomalyScanner scanner = new DefaultScanner(); + boolean shouldStart = false; + long id; + if (null == CacheUtils.cacheBuilder.getIfPresent("jobHistoryId")) { + long maxId = MapperFactory.getJobHistoryMapper().selectMaxId(); + long beginId = 0L; + if (maxId > backtrackNum) { + beginId = maxId - backtrackNum; + } + id = MapperFactory.getJobHistoryMapper().selectIdByHalfDay(beginId); + CacheUtils.cacheBuilder.put("jobHistoryId", id); + } else { + id = CacheUtils.cacheBuilder.getIfPresent("jobHistoryId"); + } + List fetchers = generateFetchersfortime(startTime, endTime, id, "updated_time"); + if (fetchers == null) { + logger.warn("generated 0 dataFetchers, plz check input"); + return; + } + // errorCode + try { + Map errorCodeAlerts = + MonitorAlertUtils.getAlerts(Constants.SCAN_PREFIX_ERRORCODE(), null); - CommonJobRunTimeRule commonJobRunTimeRule = - new CommonJobRunTimeRule(new CommonRunTimeAlertSender()); - scanner.addScanRule(commonJobRunTimeRule); - } - } catch (Exception e) { - logger.warn("Jobhistory JobResultRunTimeForDSS ErrorCode Faily: "+ e.getMessage()); - } - run(scanner, fetchers, shouldStart); + if (errorCodeAlerts == null || errorCodeAlerts.size() == 0) { + logger.info("[INFO] Loaded 0 errorcode alert from alert-rule properties file."); + } else { + logger.info( + "[INFO] Loaded {} errorcode alert from alert-rules properties file.", + errorCodeAlerts.size()); + shouldStart = true; + addIntervalToImsAlerts(errorCodeAlerts, realIntervals); + JobHistoryErrCodeRule jobHistoryErrCodeRule = + new JobHistoryErrCodeRule( + errorCodeAlerts.keySet(), new JobHistoryErrorCodeAlertSender(errorCodeAlerts)); + scanner.addScanRule(jobHistoryErrCodeRule); + } + } catch (Exception e) { + logger.warn("Jobhistory Monitor ErrorCode Faily: " + e.getMessage()); + } + // userLabel + try { + Map userLabelAlerts = + UserLabelAlertUtils.getAlerts(Constants.USER_LABEL_MONITOR(), ""); + if (userLabelAlerts == null || userLabelAlerts.size() == 0) { + logger.info("[INFO] Loaded 0 alerts userLabel alert-rule from alert properties file."); + } else { + logger.info( + "[INFO] Loaded {} alerts userLabel alert-rules from alert properties file.", + userLabelAlerts.size()); + shouldStart = true; + JobHistoryLabelsRule jobHistoryLabelsRule = + new JobHistoryLabelsRule(new JobHistoryLabelsAlertSender()); + scanner.addScanRule(jobHistoryLabelsRule); + } + } catch (Exception e) { + logger.warn("Jobhistory Monitor UserLabel Faily: " + e.getMessage()); + } + // jobResultRunTime + try { + Map jobResultAlerts = + MonitorAlertUtils.getAlerts((Constants.SCAN_PREFIX_ERRORCODE()), null); + if (jobResultAlerts == null || jobResultAlerts.size() == 0) { + logger.info("[INFO] Loaded 0 jobResult alert from alert-rule properties file."); + } else { + logger.info( + "[INFO] Loaded {} alerts jobResult alert-rules from alert properties file.", + jobResultAlerts.size()); + shouldStart = true; + JobHistoryRunTimeRule jobHistoryRunTimeRule = + new JobHistoryRunTimeRule(new JobHistoryRunTimeAlertSender()); + scanner.addScanRule(jobHistoryRunTimeRule); + } + } catch (Exception e) { + logger.warn("Jobhistory Monitor JobResultRunTime Faily: " + e.getMessage()); } + // jobResultRunTimeForDSS + try { + Map dssJobResultAlerts = + MonitorAlertUtils.getAlerts((Constants.SCAN_PREFIX_ERRORCODE()), null); + if (dssJobResultAlerts == null || dssJobResultAlerts.size() == 0) { + logger.info("[INFO] Loaded 0 jobResult alert from alert-rule properties file."); + } else { + logger.info( + "[INFO] Loaded {} alerts jobResult alert-rules from alert properties file.", + dssJobResultAlerts.size()); + shouldStart = true; - /*** - * Whether scanning data within 12 hours has timed out - */ - @Scheduled(cron = "${linkis.monitor.jobHistory.ScanTask.cron}") - public void jobHistoryUnfinishedScan() { - long id = - Optional.ofNullable(CacheUtils.cacheBuilder.getIfPresent("jobhistoryScan")) - .orElse(MonitorConfig.JOB_HISTORY_TIME_EXCEED.getValue()); - long intervalMs = Constants.SCAN_INTERVALS_SECONDS() * 1000; - long maxIntervalMs = Constants.MAX_INTERVALS_SECONDS() * 1000; - long endTime = System.currentTimeMillis(); - long startTime = endTime - intervalMs; - long realIntervals = endTime - startTime < maxIntervalMs ? endTime - startTime : maxIntervalMs; - AnomalyScanner scanner = new DefaultScanner(); - boolean shouldStart = false; - List fetchers = generateFetchers(startTime, endTime, maxIntervalMs, id, "created_time"); - if (fetchers == null) { - logger.warn("generated 0 dataFetchers, plz check input"); - return; - } - Map jobTimeAlerts = - JobHistoryScanImsAlertPropFileParserUtils.getAlerts( - Constants.SCAN_PREFIX_UNFINISHED_JOBTIME_EXCEED_SEC()); - if (jobTimeAlerts == null || jobTimeAlerts.size() == 0) { - logger.info("[INFO] Loaded 0 alerts jobtime alert-rule from alert properties file."); - } else { - logger.info( - "[INFO] Loaded {} alerts jobtime alert-rules from alert properties file.", - jobTimeAlerts.size()); - shouldStart = true; - addIntervalToImsAlerts(jobTimeAlerts, realIntervals); - JobTimeExceedRule jobTimeExceedRule = - new JobTimeExceedRule( - jobTimeAlerts.keySet(), new JobTimeExceedAlertSender(jobTimeAlerts)); - scanner.addScanRule(jobTimeExceedRule); - } - run(scanner, fetchers, shouldStart); + CommonJobRunTimeRule commonJobRunTimeRule = + new CommonJobRunTimeRule(new CommonRunTimeAlertSender()); + scanner.addScanRule(commonJobRunTimeRule); + } + } catch (Exception e) { + logger.warn("Jobhistory JobResultRunTimeForDSS ErrorCode Faily: " + e.getMessage()); } + run(scanner, fetchers, shouldStart); + } - public static void run(AnomalyScanner scanner, List fetchers, Boolean shouldStart) { - if (shouldStart) { - scanner.addDataFetchers(fetchers); - scanner.run(); - // scanner.shutdown(); // wait all alert to be send - } + /** * Whether scanning data within 12 hours has timed out */ + @Scheduled(cron = "${linkis.monitor.jobHistory.ScanTask.cron}") + public void jobHistoryUnfinishedScan() { + long id = + Optional.ofNullable(CacheUtils.cacheBuilder.getIfPresent("jobhistoryScan")) + .orElse(MonitorConfig.JOB_HISTORY_TIME_EXCEED.getValue()); + long intervalMs = Constants.ERRORCODE_SCAN_INTERVALS_SECONDS() * 1000; + long maxIntervalMs = Constants.ERRORCODE_MAX_INTERVALS_SECONDS() * 1000; + long endTime = System.currentTimeMillis(); + long startTime = endTime - intervalMs; + long realIntervals = endTime - startTime < maxIntervalMs ? endTime - startTime : maxIntervalMs; + AnomalyScanner scanner = new DefaultScanner(); + boolean shouldStart = false; + List fetchers = + generateFetchers(startTime, endTime, maxIntervalMs, id, "created_time"); + if (fetchers == null) { + logger.warn("generated 0 dataFetchers, plz check input"); + return; } + Map jobTimeAlerts = + MonitorAlertUtils.getAlerts((Constants.SCAN_PREFIX_UNFINISHED_JOBTIME_EXCEED_SEC()), null); + if (jobTimeAlerts == null || jobTimeAlerts.size() == 0) { + logger.info("[INFO] Loaded 0 alerts jobtime alert-rule from alert properties file."); + } else { + logger.info( + "[INFO] Loaded {} alerts jobtime alert-rules from alert properties file.", + jobTimeAlerts.size()); + shouldStart = true; + addIntervalToImsAlerts(jobTimeAlerts, realIntervals); + JobTimeExceedRule jobTimeExceedRule = + new JobTimeExceedRule( + jobTimeAlerts.keySet(), new JobTimeExceedAlertSender(jobTimeAlerts)); + scanner.addScanRule(jobTimeExceedRule); + } + run(scanner, fetchers, shouldStart); + } - private static List generateFetchers( - long startTime, long endTime, long maxIntervalMs, long id, String timeType) { - List ret = new ArrayList<>(); - long pe = endTime; - long ps; - while (pe > startTime) { - ps = pe - maxIntervalMs > startTime ? pe - maxIntervalMs : startTime; - String[] fetcherArgs = - new String[]{String.valueOf(ps), String.valueOf(pe), String.valueOf(id), timeType}; - ret.add(new JobHistoryDataFetcher(fetcherArgs, MapperFactory.getJobHistoryMapper())); - logger.info( - "Generated dataFetcher for startTime: " - + new Date(ps).toString() - + ". EndTime: " - + new Date(pe).toString()); - pe = pe - maxIntervalMs; - } - return ret; + public static void run(AnomalyScanner scanner, List fetchers, Boolean shouldStart) { + if (shouldStart) { + scanner.addDataFetchers(fetchers); + scanner.run(); + // scanner.shutdown(); // wait all alert to be send } + } - private static List generateFetchersfortime(long startTime, long endTime, long id, String timeType) { - List fetchers = new ArrayList<>(); - String[] fetcherArgs = - new String[]{String.valueOf(startTime), String.valueOf(endTime), String.valueOf(id), timeType}; - fetchers.add(new JobHistoryDataFetcher(fetcherArgs, MapperFactory.getJobHistoryMapper())); - logger.info( - "Generated dataFetcher for startTime: " - + new Date(startTime).toString() - + ". EndTime: " - + new Date(endTime).toString()); - return fetchers; + private static List generateFetchers( + long startTime, long endTime, long maxIntervalMs, long id, String timeType) { + List ret = new ArrayList<>(); + long pe = endTime; + long ps; + while (pe > startTime) { + ps = pe - maxIntervalMs > startTime ? pe - maxIntervalMs : startTime; + String[] fetcherArgs = + new String[] {String.valueOf(ps), String.valueOf(pe), String.valueOf(id), timeType}; + ret.add(new JobHistoryDataFetcher(fetcherArgs, MapperFactory.getJobHistoryMapper())); + logger.info( + "Generated dataFetcher for startTime: " + + new Date(ps).toString() + + ". EndTime: " + + new Date(pe).toString()); + pe = pe - maxIntervalMs; } + return ret; + } + + private static List generateFetchersfortime( + long startTime, long endTime, long id, String timeType) { + List fetchers = new ArrayList<>(); + String[] fetcherArgs = + new String[] { + String.valueOf(startTime), String.valueOf(endTime), String.valueOf(id), timeType + }; + fetchers.add(new JobHistoryDataFetcher(fetcherArgs, MapperFactory.getJobHistoryMapper())); + logger.info( + "Generated dataFetcher for startTime: " + + new Date(startTime).toString() + + ". EndTime: " + + new Date(endTime).toString()); + return fetchers; + } - private static void addIntervalToImsAlerts(Map alerts, long realIntervals) { - for (AlertDesc alert : alerts.values()) { - if (!(alert instanceof ImsAlertDesc)) { - logger.info("[warn] ignore wrong alert" + alert); - } else { - ((ImsAlertDesc) alert).hitIntervalMs_$eq(realIntervals); - } - } + private static void addIntervalToImsAlerts(Map alerts, long realIntervals) { + for (AlertDesc alert : alerts.values()) { + if (!(alert instanceof ImsAlertDesc)) { + logger.info("[warn] ignore wrong alert" + alert); + } else { + ((ImsAlertDesc) alert).hitIntervalMs_$eq(realIntervals); + } } + } } diff --git a/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/monitor/scheduled/ResourceMonitor.java b/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/monitor/scheduled/ResourceMonitor.java index b7066ba420..459aaf70ff 100644 --- a/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/monitor/scheduled/ResourceMonitor.java +++ b/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/monitor/scheduled/ResourceMonitor.java @@ -41,9 +41,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -/*** - * Monitor the usage of ECM resources for monitoring and metrics reporting - */ +/** * Monitor the usage of ECM resources for monitoring and metrics reporting */ @Component @PropertySource(value = "classpath:linkis-et-monitor.properties", encoding = "UTF-8") public class ResourceMonitor { @@ -68,7 +66,7 @@ public void ecmResourceTask() { List> emNodeVoList = data.getOrDefault("EMs", new ArrayList<>()); StringJoiner minor = new StringJoiner(","); StringJoiner major = new StringJoiner(","); - // deal ecm resource + // deal ecm resource emNodeVoList.forEach( emNodeVo -> { Map leftResource = MapUtils.getMap(emNodeVo, "leftResource"); @@ -78,8 +76,10 @@ public void ecmResourceTask() { labels.stream() .filter(labelmap -> labelmap.containsKey("tenant")) .forEach(map -> tenant.set("租户标签：" + map.get("stringValue").toString())); - String leftmemory = ByteTimeUtils.bytesToString((long) leftResource.getOrDefault("memory",0)); - String maxmemory = ByteTimeUtils.bytesToString((long) maxResource.getOrDefault("memory",0)); + String leftmemory = + ByteTimeUtils.bytesToString((long) leftResource.getOrDefault("memory", 0)); + String maxmemory = + ByteTimeUtils.bytesToString((long) maxResource.getOrDefault("memory", 0)); String leftmemoryStr = leftmemory.split(" ")[0]; String maxmemoryStr = maxmemory.split(" ")[0]; @@ -94,9 +94,12 @@ public void ecmResourceTask() { BigDecimal maxMemory = new BigDecimal(maxmemoryStr); BigDecimal maxCores = new BigDecimal((int) maxResource.get("cores")); BigDecimal maxInstance = new BigDecimal((int) maxResource.get("instance")); - double memorydouble = leftMemory.divide(maxMemory,2, BigDecimal.ROUND_HALF_DOWN).doubleValue(); - double coresdouble = leftCores.divide(maxCores,2, BigDecimal.ROUND_HALF_DOWN).doubleValue(); - double instancedouble = leftInstance.divide(maxInstance,2, BigDecimal.ROUND_HALF_DOWN).doubleValue(); + double memorydouble = + leftMemory.divide(maxMemory, 2, BigDecimal.ROUND_HALF_DOWN).doubleValue(); + double coresdouble = + leftCores.divide(maxCores, 2, BigDecimal.ROUND_HALF_DOWN).doubleValue(); + double instancedouble = + leftInstance.divide(maxInstance, 2, BigDecimal.ROUND_HALF_DOWN).doubleValue(); Double majorValue = MonitorConfig.ECM_TASK_MAJOR.getValue(); Double minorValue = MonitorConfig.ECM_TASK_MINOR.getValue(); if (((memorydouble) <= majorValue) @@ -126,26 +129,38 @@ public void ecmResourceTask() { MonitorAlertUtils.getAlerts(Constants.ALERT_RESOURCE_MONITOR(), replaceParm); PooledImsAlertUtils.addAlert(ecmResourceAlerts.get("12003")); } - // ECM资源占比上报 - resourceSendToIms(coresdouble, memorydouble, instancedouble, HttpsUntils.localHost,"USED"); - }); - //ECM 剩余资源总数上报 - resourceSendToIms(totalCores.get(), totalMemory.get(), totalInstance.get(), HttpsUntils.localHost,"TOTAL"); - } + // ECM资源占比上报 + resourceSendToIms( + coresdouble, memorydouble, instancedouble, HttpsUntils.localHost, "USED"); + }); + // ECM 剩余资源总数上报 + resourceSendToIms( + totalCores.get(), totalMemory.get(), totalInstance.get(), HttpsUntils.localHost, "TOTAL"); + } - private void resourceSendToIms(Double coresdouble, Double memorydouble, Double instancedouble, String loaclhost, String name) { - List list = new ArrayList<>(); - logger.info("ResourceMonitor send index "); - String core ="ECM_CPU_"; - String memory ="ECM_MEMORY_"; - String instance ="ECM_INSTANCE_"; - list.add(new IndexEntity(core.concat(name), "CPU", "INDEX", loaclhost, String.valueOf(coresdouble))); - list.add(new IndexEntity(memory.concat(name), "MEMORY", "INDEX", loaclhost, String.valueOf(memorydouble))); - list.add(new IndexEntity(instance.concat(name), "INSTANCE", "INDEX", loaclhost, String.valueOf(instancedouble))); - try { - HttpsUntils.sendIndex(list); - } catch (IOException e) { - logger.warn("failed to send EcmResource index"); - } + private void resourceSendToIms( + Double coresdouble, + Double memorydouble, + Double instancedouble, + String loaclhost, + String name) { + List list = new ArrayList<>(); + logger.info("ResourceMonitor send index "); + String core = "ECM_CPU_"; + String memory = "ECM_MEMORY_"; + String instance = "ECM_INSTANCE_"; + list.add( + new IndexEntity(core.concat(name), "CPU", "INDEX", loaclhost, String.valueOf(coresdouble))); + list.add( + new IndexEntity( + memory.concat(name), "MEMORY", "INDEX", loaclhost, String.valueOf(memorydouble))); + list.add( + new IndexEntity( + instance.concat(name), "INSTANCE", "INDEX", loaclhost, String.valueOf(instancedouble))); + try { + HttpsUntils.sendIndex(list); + } catch (IOException e) { + logger.warn("failed to send EcmResource index"); } + } } diff --git a/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/monitor/scheduled/TaskLogClear.java b/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/monitor/scheduled/TaskLogClear.java index bae3dc53f7..ae7e1bc21d 100644 --- a/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/monitor/scheduled/TaskLogClear.java +++ b/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/monitor/scheduled/TaskLogClear.java @@ -30,9 +30,7 @@ import org.slf4j.Logger; -/*** - * Task: clean up logs, file data of ec materials - */ +/** * Task: clean up logs, file data of ec materials */ @Component @PropertySource(value = "classpath:linkis-et-monitor.properties", encoding = "UTF-8") public class TaskLogClear { diff --git a/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/monitor/scheduled/TaskMonitor.java b/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/monitor/scheduled/TaskMonitor.java index 87bc6d6e22..5f6728ef62 100644 --- a/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/monitor/scheduled/TaskMonitor.java +++ b/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/monitor/scheduled/TaskMonitor.java @@ -43,9 +43,9 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -/*** - * Entrance monitors the number of tasks for specified users and systems. - * If the configured threshold is exceeded, an alarm will be triggered. +/** + * * Entrance monitors the number of tasks for specified users and systems. If the configured + * threshold is exceeded, an alarm will be triggered. */ @Component @PropertySource(value = "classpath:linkis-et-monitor.properties", encoding = "UTF-8") @@ -56,7 +56,6 @@ public class TaskMonitor { private static final String ENTRANCE_RUNNING_TASK = "entrance_running_task"; private static final String ENTRANCE_QUEUED_TASK = "entrance_queued_task"; - @Scheduled(cron = "${linkis.monitor.entranceTask.cron}") public void entranceTask() throws IOException { List> userlist = new ArrayList<>(); @@ -71,7 +70,8 @@ public void entranceTask() throws IOException { try { data = MapUtils.getMap( - HttpsUntils.getEntranceTask(null, entranceEntity.get("username"),null), "data"); + HttpsUntils.getEntranceTask(null, entranceEntity.get("username"), null), + "data"); logger.info("TaskMonitor userlist response {}:", data); } catch (IOException e) { logger.warn("failed to get EntranceTask data"); @@ -119,7 +119,7 @@ public void entranceTask() throws IOException { }); Map likisData = null; try { - likisData = MapUtils.getMap(HttpsUntils.getEntranceTask(null, "hadoop",null), "data"); + likisData = MapUtils.getMap(HttpsUntils.getEntranceTask(null, "hadoop", null), "data"); logger.info("TaskMonitor hadoop response {}:", likisData); } catch (IOException e) { logger.warn("failed to get EntranceTask data"); @@ -149,35 +149,49 @@ public void entranceTask() throws IOException { MonitorAlertUtils.getAlerts(Constants.ALERT_RESOURCE_MONITOR(), parms); PooledImsAlertUtils.addAlert(ecmResourceAlerts.get("12008")); } - //指标上报 + // 指标上报 resourceSendToIms(); } - public static void resourceSendToIms() { - //获取所有的entrance实例，逐个上送IMS - ServiceInstance[] instances = Sender.getInstances(Constants.DIRTY_DATA_ENTRANCE_APPLICATIONNAME()); - if (null != instances) { - for (ServiceInstance instance : instances) { - String serviceInstance = instance.getInstance(); - try { - Map instanceData = MapUtils.getMap(HttpsUntils.getEntranceTask(null, "hadoop", serviceInstance), "data"); - int runningNumber = 0; - int queuedNumber = 0; - if (instanceData.containsKey("runningNumber")) { - runningNumber = (int) instanceData.get("runningNumber"); - } - if (instanceData.containsKey("queuedNumber")) { - queuedNumber = (int) instanceData.get("queuedNumber"); - } - logger.info("ResourceMonitor send index "); - List list = new ArrayList<>(); - list.add(new IndexEntity(serviceInstance, "entrance", ENTRANCE_RUNNING_TASK, HttpsUntils.localHost, String.valueOf(runningNumber))); - list.add(new IndexEntity(serviceInstance, "entrance", ENTRANCE_QUEUED_TASK, HttpsUntils.localHost, String.valueOf(queuedNumber))); - HttpsUntils.sendIndex(list); - } catch (IOException e) { - logger.warn("failed to send EcmResource index :" + e); - } - } + public static void resourceSendToIms() { + // 获取所有的entrance实例，逐个上送IMS + ServiceInstance[] instances = + Sender.getInstances(Constants.DIRTY_DATA_ENTRANCE_APPLICATIONNAME()); + if (null != instances) { + for (ServiceInstance instance : instances) { + String serviceInstance = instance.getInstance(); + try { + Map instanceData = + MapUtils.getMap(HttpsUntils.getEntranceTask(null, "hadoop", serviceInstance), "data"); + int runningNumber = 0; + int queuedNumber = 0; + if (instanceData.containsKey("runningNumber")) { + runningNumber = (int) instanceData.get("runningNumber"); + } + if (instanceData.containsKey("queuedNumber")) { + queuedNumber = (int) instanceData.get("queuedNumber"); + } + logger.info("ResourceMonitor send index "); + List list = new ArrayList<>(); + list.add( + new IndexEntity( + serviceInstance, + "entrance", + ENTRANCE_RUNNING_TASK, + HttpsUntils.localHost, + String.valueOf(runningNumber))); + list.add( + new IndexEntity( + serviceInstance, + "entrance", + ENTRANCE_QUEUED_TASK, + HttpsUntils.localHost, + String.valueOf(queuedNumber))); + HttpsUntils.sendIndex(list); + } catch (IOException e) { + logger.warn("failed to send EcmResource index :" + e); } + } } + } } diff --git a/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/monitor/scheduled/UserModeMonitor.java b/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/monitor/scheduled/UserModeMonitor.java index 03db536344..c642a24762 100644 --- a/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/monitor/scheduled/UserModeMonitor.java +++ b/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/monitor/scheduled/UserModeMonitor.java @@ -35,8 +35,10 @@ import org.apache.linkis.ujes.client.response.GetTableStatisticInfoResult; import org.apache.linkis.ujes.client.response.JobExecuteResult; import org.apache.linkis.ujes.client.response.JobInfoResult; + import org.springframework.scheduling.annotation.Scheduled; import org.springframework.stereotype.Component; + import java.net.SocketTimeoutException; import java.util.*; import java.util.concurrent.TimeUnit; @@ -45,112 +47,130 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -/*** - * User mode monitoring: regularly trigger scripts to monitor whether the engine status is running normally +/** + * * User mode monitoring: regularly trigger scripts to monitor whether the engine status is running + * normally */ @Component public class UserModeMonitor { - private static final Logger logger = LoggerFactory.getLogger(UserModeMonitor.class); + private static final Logger logger = LoggerFactory.getLogger(UserModeMonitor.class); - private static final DWSClientConfig clientConfig = HttpsUntils.dwsClientConfig; + private static final DWSClientConfig clientConfig = HttpsUntils.dwsClientConfig; - private static final UJESClient client = new UJESClientImpl(clientConfig); + private static final UJESClient client = new UJESClientImpl(clientConfig); - @Scheduled(cron = "${linkis.monitor.user.cron}") - public void job() { - Optional.ofNullable(MonitorConfig.USER_MODE_ENGINE.getValue()).ifPresent(configStr -> { - ArrayList> userModeStr = - BDPJettyServerHelper.gson().fromJson(configStr, ArrayList.class); - userModeStr.forEach(engine -> { - // 3. build job and execute - JobExecuteResult jobExecuteResult = toSubmit(engine); + @Scheduled(cron = "${linkis.monitor.user.cron}") + public void job() { + Optional.ofNullable(MonitorConfig.USER_MODE_ENGINE.getValue()) + .ifPresent( + configStr -> { + ArrayList> userModeStr = + BDPJettyServerHelper.gson().fromJson(configStr, ArrayList.class); + userModeStr.forEach( + engine -> { + // 3. build job and execute + JobExecuteResult jobExecuteResult = toSubmit(engine); + logger.info( + "start run engineType: {},job id : {}", + engine.get("engineType"), + jobExecuteResult.taskID()); + HashMap parms = new HashMap<>(); + parms.put("$engineType", engine.get("engineType")); + parms.put("$url", MonitorConfig.GATEWAY_URL.getValue()); + parms.put("$jobId", jobExecuteResult.taskID()); + Utils.sleepQuietly(MonitorConfig.USER_MODE_TIMEOUT.getValue() * 1000); + JobInfoResult jobInfo = client.getJobInfo(jobExecuteResult); + if (jobInfo.isCompleted()) { + if (jobInfo.getJobStatus().equals("Failed")) { logger.info( - "start run engineType: {},job id : {}", - engine.get("engineType"), - jobExecuteResult.taskID()); - HashMap parms = new HashMap<>(); - parms.put("$engineType", engine.get("engineType")); - parms.put("$url", MonitorConfig.GATEWAY_URL.getValue()); - parms.put("$jobId", jobExecuteResult.taskID()); - Utils.sleepQuietly(MonitorConfig.USER_MODE_TIMEOUT.getValue() * 1000); - JobInfoResult jobInfo = client.getJobInfo(jobExecuteResult); - if (jobInfo.isCompleted()) { - if (jobInfo.getJobStatus().equals("Failed")) { - logger.info("run fail engineType: {},job id : {}", engine.get("engineType"), jobExecuteResult.taskID()); - RequestPersistTask requestPersistTask = jobInfo.getRequestPersistTask(); - parms.put("$errorCode", String.valueOf(requestPersistTask.getErrCode())); - parms.put("$errorMsg", requestPersistTask.getErrDesc()); - Map failedAlerts = MonitorAlertUtils.getAlerts(Constants.USER_RESOURCE_MONITOR(), parms); - PooledImsAlertUtils.addAlert(failedAlerts.get("12012")); - } - } else { - logger.info("run timeout engineType: {},job id : {}", engine.get("engineType"), jobExecuteResult.taskID()); - Map alerts = MonitorAlertUtils.getAlerts(Constants.USER_RESOURCE_MONITOR(), parms); - PooledImsAlertUtils.addAlert(alerts.get("12011")); - } + "run fail engineType: {},job id : {}", + engine.get("engineType"), + jobExecuteResult.taskID()); + RequestPersistTask requestPersistTask = jobInfo.getRequestPersistTask(); + parms.put("$errorCode", String.valueOf(requestPersistTask.getErrCode())); + parms.put("$errorMsg", requestPersistTask.getErrDesc()); + Map failedAlerts = + MonitorAlertUtils.getAlerts(Constants.USER_RESOURCE_MONITOR(), parms); + PooledImsAlertUtils.addAlert(failedAlerts.get("12012")); + } + } else { + logger.info( + "run timeout engineType: {},job id : {}", + engine.get("engineType"), + jobExecuteResult.taskID()); + Map alerts = + MonitorAlertUtils.getAlerts(Constants.USER_RESOURCE_MONITOR(), parms); + PooledImsAlertUtils.addAlert(alerts.get("12011")); } - ); - }); - } + }); + }); + } - private static JobExecuteResult toSubmit(LinkedTreeMap engine) { - // 1. build params - // set label map :EngineTypeLabel/UserCreatorLabel/EngineRunTypeLabel/Tenant - Map labels = new HashMap(); - labels.put(LabelKeyConstant.ENGINE_TYPE_KEY, engine.get("engineType")); // required engineType Label - labels.put(LabelKeyConstant.USER_CREATOR_TYPE_KEY, engine.get("executeUser") + "-IDE");// required execute user and creator eg:hadoop-IDE - labels.put(LabelKeyConstant.CODE_TYPE_KEY, engine.get("runType")); // required codeType - Map startupMap = new HashMap(16); - // setting linkis params - //startupMap.put("wds.linkis.rm.yarnqueue", "dws"); - // 2. build jobSubmitAction - JobSubmitAction jobSubmitAction = JobSubmitAction.builder() - .addExecuteCode(engine.get("code")) - .setStartupParams(startupMap) - .setUser(engine.get("executeUser")) //submit user - .addExecuteUser(engine.get("executeUser")) // execute user - .setLabels(labels) - .build(); - // 3. to execute - return client.submit(jobSubmitAction); - } + private static JobExecuteResult toSubmit(LinkedTreeMap engine) { + // 1. build params + // set label map :EngineTypeLabel/UserCreatorLabel/EngineRunTypeLabel/Tenant + Map labels = new HashMap(); + labels.put( + LabelKeyConstant.ENGINE_TYPE_KEY, engine.get("engineType")); // required engineType Label + labels.put( + LabelKeyConstant.USER_CREATOR_TYPE_KEY, + engine.get("executeUser") + "-IDE"); // required execute user and creator eg:hadoop-IDE + labels.put(LabelKeyConstant.CODE_TYPE_KEY, engine.get("runType")); // required codeType + Map startupMap = new HashMap(16); + // setting linkis params + // startupMap.put("wds.linkis.rm.yarnqueue", "dws"); + // 2. build jobSubmitAction + JobSubmitAction jobSubmitAction = + JobSubmitAction.builder() + .addExecuteCode(engine.get("code")) + .setStartupParams(startupMap) + .setUser(engine.get("executeUser")) // submit user + .addExecuteUser(engine.get("executeUser")) // execute user + .setLabels(labels) + .build(); + // 3. to execute + return client.submit(jobSubmitAction); + } - @Scheduled(cron = "${linkis.monitor.user.db.cron:0 0/10 * * * ?}") - public void dbJob() { - Map properties= new HashMap<>(); - properties.put("readTimeout",MonitorConfig.USER_MODE_INTERFACE_TIMEOUT.getValue()); - DWSClientConfig clientConfig = HttpsUntils.createClientConfig(null, properties); - UJESClientImpl ujesClient = new UJESClientImpl(clientConfig); - GetTableStatisticInfoAction builder = GetTableStatisticInfoAction - .builder() - .setUser("hadoop") - .setDatabase("default") - .setTable("dual") - .builder(); - HashMap parms = new HashMap<>(); - try { - GetTableStatisticInfoResult tableStatisticInfo = ujesClient.getTableStatisticInfo(builder); - if (tableStatisticInfo.getStatus() != 0) { - logger.info("元数据查询服务用户态，执行失败，异常信息："+tableStatisticInfo.getMessage()); -// parms.put("$msg", tableStatisticInfo.getMessage()); -// Map failedAlerts = MonitorAlertUtils.getAlerts(Constants.USER_RESOURCE_MONITOR(), parms); -// PooledImsAlertUtils.addAlert(failedAlerts.get("12017")); - } - } catch (Exception e) { - if(e instanceof SocketTimeoutException){ - Integer timeoutValue = MonitorConfig.USER_MODE_INTERFACE_TIMEOUT.getValue(); - long timeout = TimeUnit.MILLISECONDS.toSeconds(timeoutValue); - logger.info("元数据查询服务用户态，执行超时："+timeout+"秒"); -// parms.put("$timeout", String.valueOf(timeout)); -// Map failedAlerts = MonitorAlertUtils.getAlerts(Constants.USER_RESOURCE_MONITOR(), parms); -// PooledImsAlertUtils.addAlert(failedAlerts.get("12018")); - } else { - logger.error("元数据查询服务用户态，执行异常："+ e); -// parms.put("$msg", e.getMessage()); -// Map failedAlerts = MonitorAlertUtils.getAlerts(Constants.USER_RESOURCE_MONITOR(), parms); -// PooledImsAlertUtils.addAlert(failedAlerts.get("12017")); - } - } + @Scheduled(cron = "${linkis.monitor.user.db.cron:0 0/10 * * * ?}") + public void dbJob() { + Map properties = new HashMap<>(); + properties.put("readTimeout", MonitorConfig.USER_MODE_INTERFACE_TIMEOUT.getValue()); + DWSClientConfig clientConfig = HttpsUntils.createClientConfig(null, properties); + UJESClientImpl ujesClient = new UJESClientImpl(clientConfig); + GetTableStatisticInfoAction builder = + GetTableStatisticInfoAction.builder() + .setUser("hadoop") + .setDatabase("default") + .setTable("dual") + .builder(); + HashMap parms = new HashMap<>(); + try { + GetTableStatisticInfoResult tableStatisticInfo = ujesClient.getTableStatisticInfo(builder); + if (tableStatisticInfo.getStatus() != 0) { + logger.info("元数据查询服务用户态，执行失败，异常信息：" + tableStatisticInfo.getMessage()); + // parms.put("$msg", tableStatisticInfo.getMessage()); + // Map failedAlerts = + // MonitorAlertUtils.getAlerts(Constants.USER_RESOURCE_MONITOR(), parms); + // PooledImsAlertUtils.addAlert(failedAlerts.get("12017")); + } + } catch (Exception e) { + if (e instanceof SocketTimeoutException) { + Integer timeoutValue = MonitorConfig.USER_MODE_INTERFACE_TIMEOUT.getValue(); + long timeout = TimeUnit.MILLISECONDS.toSeconds(timeoutValue); + logger.info("元数据查询服务用户态，执行超时：" + timeout + "秒"); + // parms.put("$timeout", String.valueOf(timeout)); + // Map failedAlerts = + // MonitorAlertUtils.getAlerts(Constants.USER_RESOURCE_MONITOR(), parms); + // PooledImsAlertUtils.addAlert(failedAlerts.get("12018")); + } else { + logger.error("元数据查询服务用户态，执行异常：" + e); + // parms.put("$msg", e.getMessage()); + // Map failedAlerts = + // MonitorAlertUtils.getAlerts(Constants.USER_RESOURCE_MONITOR(), parms); + // PooledImsAlertUtils.addAlert(failedAlerts.get("12017")); + } } + } } diff --git a/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/monitor/scheduled/ValidatorClear.java b/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/monitor/scheduled/ValidatorClear.java index 440b7a6bc4..4df828e36c 100644 --- a/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/monitor/scheduled/ValidatorClear.java +++ b/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/monitor/scheduled/ValidatorClear.java @@ -20,32 +20,32 @@ import org.apache.linkis.monitor.scan.app.monitor.config.MonitorConfig; import org.apache.linkis.monitor.scan.app.monitor.until.ThreadUtils; import org.apache.linkis.monitor.scan.utils.log.LogUtils; -import org.slf4j.Logger; + import org.springframework.context.annotation.PropertySource; import org.springframework.scheduling.annotation.Scheduled; import org.springframework.stereotype.Component; import java.util.ArrayList; import java.util.List; -/*** - * Task: clean up linkis_et_validator_checkinfo data - */ +import org.slf4j.Logger; + +/** * Task: clean up linkis_et_validator_checkinfo data */ @Component @PropertySource(value = "classpath:linkis-et-monitor.properties", encoding = "UTF-8") public class ValidatorClear { - private static final Logger logger = LogUtils.stdOutLogger(); - - @Scheduled(cron = "${linkis.monitor.clear.validator.cron}") - public void ValidatorClear() { - logger.info("Start to clear_validator_record shell"); - List cmdlist = new ArrayList<>(); - cmdlist.add("sh"); - cmdlist.add(MonitorConfig.shellPath + "clear_validator_record.sh"); - logger.info("clear_validator_record shell command {}", cmdlist); - String exec = ThreadUtils.run(cmdlist, "clear_validator_record.sh"); - logger.info("shell log {}", exec); - logger.info("End to clear_validator_record shell "); - } + private static final Logger logger = LogUtils.stdOutLogger(); + + @Scheduled(cron = "${linkis.monitor.clear.validator.cron}") + public void ValidatorClear() { + logger.info("Start to clear_validator_record shell"); + List cmdlist = new ArrayList<>(); + cmdlist.add("sh"); + cmdlist.add(MonitorConfig.shellPath + "clear_validator_record.sh"); + logger.info("clear_validator_record shell command {}", cmdlist); + String exec = ThreadUtils.run(cmdlist, "clear_validator_record.sh"); + logger.info("shell log {}", exec); + logger.info("End to clear_validator_record shell "); + } } diff --git a/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/monitor/until/HttpsUntils.java b/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/monitor/until/HttpsUntils.java index 1208a6ef64..34e12d175d 100644 --- a/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/monitor/until/HttpsUntils.java +++ b/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/monitor/until/HttpsUntils.java @@ -32,6 +32,7 @@ import org.apache.linkis.monitor.scan.request.EntranceTaskAction; import org.apache.linkis.monitor.scan.response.EntranceTaskResult; import org.apache.linkis.server.BDPJettyServerHelper; +import org.apache.linkis.ujes.client.response.EmsListResult; import org.apache.commons.collections.MapUtils; import org.apache.commons.lang3.StringUtils; @@ -44,7 +45,6 @@ import org.apache.http.impl.client.HttpClients; import org.apache.http.util.EntityUtils; -import org.apache.linkis.ujes.client.response.EmsListResult; import org.springframework.util.Assert; import java.io.IOException; @@ -132,14 +132,16 @@ public static DWSClientConfig createClientConfig(String url, Map return clientConfig; } - public static Map getEntranceTask(String url, String user,String Instance) throws IOException { + public static Map getEntranceTask(String url, String user, String Instance) + throws IOException { if (null == dwsClientConfig) { dwsClientConfig = createClientConfig(null, null); } if (null == client) { client = new MonitorHTTPClientClientImpl(dwsClientConfig); } - EntranceTaskAction build = EntranceTaskAction.newBuilder().setUser(user).setInstance(Instance).build(); + EntranceTaskAction build = + EntranceTaskAction.newBuilder().setUser(user).setInstance(Instance).build(); EntranceTaskResult result = client.entranList(build); return result.getResultMap(); } @@ -170,7 +172,9 @@ public static void sendIndex(List list) throws IOException { public static void sendChatbot(ChatbotEntity chatbotEntity) throws IOException { String json = BDPJettyServerHelper.gson().toJson(chatbotEntity); - StringEntity entity = new StringEntity(json, ContentType.create(ContentType.APPLICATION_JSON.getMimeType(), "UTF-8")); + StringEntity entity = + new StringEntity( + json, ContentType.create(ContentType.APPLICATION_JSON.getMimeType(), "UTF-8")); entity.setContentEncoding("UTF-8"); HttpPost httpPost = new HttpPost(MonitorConfig.CHATBOT_URL.getValue()); httpPost.setConfig(RequestConfig.DEFAULT); @@ -179,5 +183,4 @@ public static void sendChatbot(ChatbotEntity chatbotEntity) throws IOException { String responseStr = EntityUtils.toString(execute.getEntity(), "UTF-8"); Map map = BDPJettyServerHelper.gson().fromJson(responseStr, Map.class); } - } diff --git a/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/app/LinkisJobHistoryScanSpringConfiguration.scala b/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/app/LinkisJobHistoryScanSpringConfiguration.scala index 7ee159d936..520c0a78c3 100644 --- a/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/app/LinkisJobHistoryScanSpringConfiguration.scala +++ b/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/app/LinkisJobHistoryScanSpringConfiguration.scala @@ -26,10 +26,6 @@ import org.springframework.context.annotation.{ComponentScan, Configuration} import javax.annotation.PostConstruct -/** - * Created by shangda on 2021/11/19. - */ - @Configuration @ComponentScan(Array("org.apache.linkis.monitor.scan", "org.apache.linkis.mybatis")) class LinkisJobHistoryScanSpringConfiguration { diff --git a/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/app/factory/MapperFactory.scala b/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/app/factory/MapperFactory.scala index 337592bf72..c7ebf10e88 100644 --- a/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/app/factory/MapperFactory.scala +++ b/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/app/factory/MapperFactory.scala @@ -22,12 +22,9 @@ import org.apache.linkis.monitor.scan.app.instance.dao.{ InstanceInfoDao, InstanceLabelDao } -import org.apache.linkis.monitor.scan.app.instance.dao.InstanceInfoDao import org.apache.linkis.monitor.scan.app.jobhistory.dao.JobHistoryMapper - object MapperFactory { - // val bmlVersionCleanScanOper = new BmlVersionCleanScanOper private var jobHistoryMapper: JobHistoryMapper = _ @@ -37,27 +34,27 @@ object MapperFactory { private var instanceLabelRelationMapper: InsLabelRelationDao = _ - def getJobHistoryMapper() = jobHistoryMapper + def getJobHistoryMapper(): JobHistoryMapper = jobHistoryMapper - def setJobHistoryMapper(jobHistoryMapper: JobHistoryMapper) = { + def setJobHistoryMapper(jobHistoryMapper: JobHistoryMapper): Unit = { MapperFactory.jobHistoryMapper = jobHistoryMapper } - def getInstanceInfoMapper() = instanceInfoMapper + def getInstanceInfoMapper(): InstanceInfoDao = instanceInfoMapper - def setInstanceInfoMapper(instanceInfoMapper: InstanceInfoDao) = { + def setInstanceInfoMapper(instanceInfoMapper: InstanceInfoDao): Unit = { MapperFactory.instanceInfoMapper = instanceInfoMapper } - def getInstanceLabelMapper() = instanceLabelMapper + def getInstanceLabelMapper(): InstanceLabelDao = instanceLabelMapper - def setInstanceLabelMapper(instanceLabelMapper: InstanceLabelDao) = { + def setInstanceLabelMapper(instanceLabelMapper: InstanceLabelDao): Unit = { MapperFactory.instanceLabelMapper = instanceLabelMapper } - def getInsLabelRelationMapper() = instanceLabelRelationMapper + def getInsLabelRelationMapper(): InsLabelRelationDao = instanceLabelRelationMapper - def setInsLabelRelationMapper(instanceLabelRelationMapper: InsLabelRelationDao) = { + def setInsLabelRelationMapper(instanceLabelRelationMapper: InsLabelRelationDao): Unit = { MapperFactory.instanceLabelRelationMapper = instanceLabelRelationMapper } diff --git a/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/app/jobhistory/JobHistoryDataFetcher.scala b/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/app/jobhistory/JobHistoryDataFetcher.scala index b8eff63ec8..c7fd3f40e8 100644 --- a/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/app/jobhistory/JobHistoryDataFetcher.scala +++ b/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/app/jobhistory/JobHistoryDataFetcher.scala @@ -17,16 +17,16 @@ package org.apache.linkis.monitor.scan.app.jobhistory -import java.util -import java.util.Date - -import org.apache.commons.lang3.StringUtils import org.apache.linkis.common.utils.{Logging, Utils} import org.apache.linkis.monitor.scan.app.jobhistory.dao.JobHistoryMapper import org.apache.linkis.monitor.scan.app.jobhistory.exception.AnomalyScannerException import org.apache.linkis.monitor.scan.constants.Constants import org.apache.linkis.monitor.scan.core.pac.AbstractDataFetcher +import org.apache.commons.lang3.StringUtils + +import java.util +import java.util.Date class JobHistoryDataFetcher(args: Array[Any], mapper: JobHistoryMapper) extends AbstractDataFetcher @@ -89,7 +89,11 @@ class JobHistoryDataFetcher(args: Array[Any], mapper: JobHistoryMapper) throw t } } - if (StringUtils.isNotBlank(args(3).asInstanceOf[String]) && args(3).asInstanceOf[String].equals("updated_time")) { + if ( + StringUtils.isNotBlank(args(3).asInstanceOf[String]) && args(3) + .asInstanceOf[String] + .equals("updated_time") + ) { val list = new util.ArrayList[String]() Constants.DIRTY_DATA_FINISHED_JOB_STATUS_ARRAY.foreach(list.add(_)) mapper diff --git a/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/app/jobhistory/errorcode/JobHistoryErrCodeHitEvent.scala b/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/app/jobhistory/errorcode/JobHistoryErrCodeHitEvent.scala index f899952c71..f89c680624 100644 --- a/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/app/jobhistory/errorcode/JobHistoryErrCodeHitEvent.scala +++ b/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/app/jobhistory/errorcode/JobHistoryErrCodeHitEvent.scala @@ -19,5 +19,4 @@ package org.apache.linkis.monitor.scan.app.jobhistory.errorcode import org.apache.linkis.monitor.scan.core.ob.SingleObserverEvent - class JobHistoryErrCodeHitEvent extends SingleObserverEvent diff --git a/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/app/jobhistory/errorcode/JobHistoryErrCodeRule.scala b/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/app/jobhistory/errorcode/JobHistoryErrCodeRule.scala index c613b3d306..32a7cbe474 100644 --- a/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/app/jobhistory/errorcode/JobHistoryErrCodeRule.scala +++ b/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/app/jobhistory/errorcode/JobHistoryErrCodeRule.scala @@ -17,19 +17,21 @@ package org.apache.linkis.monitor.scan.app.jobhistory.errorcode -import java.util - import org.apache.linkis.common.utils.Logging import org.apache.linkis.monitor.scan.app.jobhistory.entity.JobHistory import org.apache.linkis.monitor.scan.app.monitor.until.CacheUtils import org.apache.linkis.monitor.scan.core.ob.Observer import org.apache.linkis.monitor.scan.core.pac.{AbstractScanRule, ScannedData} -import scala.collection.JavaConverters._ +import java.util + +import scala.collection.JavaConverters._ /** - * 针对执行任务返回的错误码进行监控，执行脚本任务时，会记录执行的错误码在数据库中， - * 服务会根据数据库中记录的错误码，来进行告警，如果错误码中包含（11001，11002）即可触发告警 + * Monitor the error codes returned by executing tasks. When executing script tasks, the executed + * error codes will be recorded in the database. The service will generate an alarm based on the + * error code recorded in the database. If the error code contains (11001, 11002), the alarm will be + * triggered. */ class JobHistoryErrCodeRule(errorCodes: util.Set[String], hitObserver: Observer) extends AbstractScanRule(event = new JobHistoryErrCodeHitEvent, observer = hitObserver) @@ -60,7 +62,9 @@ class JobHistoryErrCodeRule(errorCodes: util.Set[String], hitObserver: Observer) } scanRuleList.put("jobHistoryId", history.getId) case _ => - logger.warn("Ignored wrong input data Type : " + d + ", " + d.getClass.getCanonicalName) + logger.warn( + "Ignored wrong input data Type : " + d + ", " + d.getClass.getCanonicalName + ) } } } else { @@ -71,9 +75,9 @@ class JobHistoryErrCodeRule(errorCodes: util.Set[String], hitObserver: Observer) logger.info("hit " + alertData.size() + " data in one iteration") if (alertData.size() > 0) { getHitEvent().notifyObserver(getHitEvent(), alertData) - true + true } else { - false + false } } diff --git a/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/app/jobhistory/errorcode/JobHistoryErrorCodeAlertSender.scala b/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/app/jobhistory/errorcode/JobHistoryErrorCodeAlertSender.scala index a037e0f017..8683dad15e 100644 --- a/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/app/jobhistory/errorcode/JobHistoryErrorCodeAlertSender.scala +++ b/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/app/jobhistory/errorcode/JobHistoryErrorCodeAlertSender.scala @@ -17,17 +17,16 @@ package org.apache.linkis.monitor.scan.app.jobhistory.errorcode -import java.util - import org.apache.linkis.common.utils.Logging import org.apache.linkis.monitor.scan.app.jobhistory.entity.JobHistory import org.apache.linkis.monitor.scan.app.jobhistory.exception.AnomalyScannerException import org.apache.linkis.monitor.scan.core.ob.{Event, Observer} import org.apache.linkis.monitor.scan.utils.alert.AlertDesc import org.apache.linkis.monitor.scan.utils.alert.ims.{ImsAlertDesc, PooledImsAlertUtils} -import scala.collection.JavaConverters._ +import java.util +import scala.collection.JavaConverters._ class JobHistoryErrorCodeAlertSender(alerts: util.Map[String, AlertDesc]) extends Observer diff --git a/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/app/jobhistory/jobtime/JobTimeExceedAlertSender.scala b/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/app/jobhistory/jobtime/JobTimeExceedAlertSender.scala index aa564ab335..fd311e7fcc 100644 --- a/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/app/jobhistory/jobtime/JobTimeExceedAlertSender.scala +++ b/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/app/jobhistory/jobtime/JobTimeExceedAlertSender.scala @@ -17,9 +17,6 @@ package org.apache.linkis.monitor.scan.app.jobhistory.jobtime -import java.text.MessageFormat -import java.util - import org.apache.linkis.common.utils.{Logging, Utils} import org.apache.linkis.monitor.scan.app.jobhistory.entity.JobHistory import org.apache.linkis.monitor.scan.app.jobhistory.exception.AnomalyScannerException @@ -28,13 +25,13 @@ import org.apache.linkis.monitor.scan.core.ob.{Event, Observer} import org.apache.linkis.monitor.scan.utils.alert.AlertDesc import org.apache.linkis.monitor.scan.utils.alert.ims.{ImsAlertDesc, PooledImsAlertUtils} +import java.text.MessageFormat +import java.util + import scala.collection.JavaConverters._ import scala.collection.mutable.ArrayBuffer - -class JobTimeExceedAlertSender(alerts: util.Map[String, AlertDesc]) - extends Observer - with Logging { +class JobTimeExceedAlertSender(alerts: util.Map[String, AlertDesc]) extends Observer with Logging { private val orderedThresholds: Array[Long] = { val ret = new ArrayBuffer[Long]() @@ -79,34 +76,36 @@ class JobTimeExceedAlertSender(alerts: util.Map[String, AlertDesc]) for (t <- orderedThresholds) { // search max threshold that is smaller than elapse if (elapse >= t) { ts = t - } else { - - } + } else {} } val name = ts.toString val alert = if (!toSend.containsKey(name)) { alerts .get(name) - .asInstanceOf[ - ImsAlertDesc - ] + .asInstanceOf[ImsAlertDesc] } else { toSend.get(name) } - - val newInfo = MessageFormat.format("[Linkis任务信息]您好，您在Linkis/DSS提交的任务(任务ID:{0})，已经运行超过{1}h，" + - "请关注是否任务正常，如果不正常您可以到Linkis/DSS管理台进行任务的kill，集群信息为BDAP({2})。详细解决方案见Q47：{3} " - , jobHistory.getId, (elapse / 1000 / 60 / 60).toString, jobHistory.getInstances, MonitorConfig.SOLUTION_URL.getValue) + + val newInfo = MessageFormat.format( + MonitorConfig.TASK_RUNTIME_TIMEOUT_DESC.getValue, + jobHistory.getId, + (elapse / 1000 / 60 / 60).toString, + jobHistory.getInstances, + MonitorConfig.SOLUTION_URL.getValue + ) val newNumHit = alert.numHit + 1 val receiver = new util.HashSet[String]() receiver.add(jobHistory.getSubmitUser) receiver.add(jobHistory.getExecuteUser) receiver.addAll(alert.alertReceivers) - val ImsAlertDesc = alert.copy(alertInfo = newInfo, alertReceivers = receiver, numHit = newNumHit) + val ImsAlertDesc = + alert.copy(alertInfo = newInfo, alertReceivers = receiver, numHit = newNumHit) PooledImsAlertUtils.addAlert(ImsAlertDesc) } } } + } diff --git a/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/app/jobhistory/jobtime/JobTimeExceedHitEvent.scala b/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/app/jobhistory/jobtime/JobTimeExceedHitEvent.scala index b7b883e09b..a83cd0ee01 100644 --- a/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/app/jobhistory/jobtime/JobTimeExceedHitEvent.scala +++ b/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/app/jobhistory/jobtime/JobTimeExceedHitEvent.scala @@ -19,5 +19,4 @@ package org.apache.linkis.monitor.scan.app.jobhistory.jobtime import org.apache.linkis.monitor.scan.core.ob.SingleObserverEvent - class JobTimeExceedHitEvent extends SingleObserverEvent diff --git a/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/app/jobhistory/jobtime/JobTimeExceedRule.scala b/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/app/jobhistory/jobtime/JobTimeExceedRule.scala index b91d605d4e..821a368d65 100644 --- a/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/app/jobhistory/jobtime/JobTimeExceedRule.scala +++ b/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/app/jobhistory/jobtime/JobTimeExceedRule.scala @@ -17,9 +17,6 @@ package org.apache.linkis.monitor.scan.app.jobhistory.jobtime -import java.util -import java.util.Locale - import org.apache.linkis.common.utils.Logging import org.apache.linkis.monitor.scan.app.jobhistory.entity.JobHistory import org.apache.linkis.monitor.scan.app.jobhistory.exception.AnomalyScannerException @@ -28,11 +25,15 @@ import org.apache.linkis.monitor.scan.constants.Constants import org.apache.linkis.monitor.scan.core.ob.Observer import org.apache.linkis.monitor.scan.core.pac.{AbstractScanRule, ScannedData} +import java.util +import java.util.Locale + import scala.collection.JavaConverters._ /** - * 针对执行任务状态进行监控，扫描12小时之外，24小时之内的的数据， - * 如果规则范围内，有数据状态是（Inited,WaitForRetry,Scheduled,Running）其中之一，则触发告警 + * Monitor the execution status of tasks, scan data outside 12 hours and within 24 hours, If within + * the scope of the rule, there is data whose status is one of (Inited, WaitForRetry, Scheduled, + * Running), an alarm will be triggered. */ class JobTimeExceedRule(thresholds: util.Set[String], hitObserver: Observer) extends AbstractScanRule(event = new JobTimeExceedHitEvent, observer = hitObserver) @@ -94,9 +95,9 @@ class JobTimeExceedRule(thresholds: util.Set[String], hitObserver: Observer) logger.info("hit " + alertData.size() + " data in one iteration") if (alertData.size() > 0) { getHitEvent.notifyObserver(getHitEvent, alertData) - true + true } else { - false + false } } diff --git a/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/app/jobhistory/labels/JobHistoryLabelsAlertSender.scala b/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/app/jobhistory/labels/JobHistoryLabelsAlertSender.scala index 77bc29ffe2..e7ad384e56 100644 --- a/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/app/jobhistory/labels/JobHistoryLabelsAlertSender.scala +++ b/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/app/jobhistory/labels/JobHistoryLabelsAlertSender.scala @@ -17,8 +17,6 @@ package org.apache.linkis.monitor.scan.app.jobhistory.labels -import java.util - import org.apache.linkis.common.utils.Logging import org.apache.linkis.monitor.scan.app.jobhistory.entity.JobHistory import org.apache.linkis.monitor.scan.app.jobhistory.exception.AnomalyScannerException @@ -28,6 +26,8 @@ import org.apache.linkis.monitor.scan.utils.alert.AlertDesc import org.apache.linkis.monitor.scan.utils.alert.ims.{PooledImsAlertUtils, UserLabelAlertUtils} import org.apache.linkis.server.BDPJettyServerHelper +import java.util + import scala.collection.JavaConverters._ import scala.collection.mutable.ArrayBuffer diff --git a/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/app/jobhistory/labels/JobHistoryLabelsRule.scala b/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/app/jobhistory/labels/JobHistoryLabelsRule.scala index 6308ac28b0..f4c65f7ba4 100644 --- a/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/app/jobhistory/labels/JobHistoryLabelsRule.scala +++ b/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/app/jobhistory/labels/JobHistoryLabelsRule.scala @@ -17,10 +17,6 @@ package org.apache.linkis.monitor.scan.app.jobhistory.labels -import java.util - -import com.google.common.collect.HashBiMap -import org.apache.commons.lang3.StringUtils import org.apache.linkis.common.utils.Logging import org.apache.linkis.monitor.scan.app.jobhistory.entity.JobHistory import org.apache.linkis.monitor.scan.app.monitor.until.CacheUtils @@ -29,11 +25,17 @@ import org.apache.linkis.monitor.scan.core.ob.Observer import org.apache.linkis.monitor.scan.core.pac.{AbstractScanRule, ScannedData} import org.apache.linkis.server.BDPJettyServerHelper +import org.apache.commons.lang3.StringUtils + +import java.util + import scala.collection.JavaConverters._ +import com.google.common.collect.HashBiMap + /** - * 对前20分钟内的执行数据进行扫描，对数据的labels字段进行判断, - * 判断依据monitor配置（linkis.monitor.jobhistory.userLabel.tenant） + * Scan the execution data within the previous 20 minutes and judge the labels field of the data. + * Judgment based on monitor configuration (linkis.monitor.jobhistory.userLabel.tenant) */ class JobHistoryLabelsRule(hitObserver: Observer) extends AbstractScanRule(event = new JobHistoryLabelsHitEvent, observer = hitObserver) @@ -69,7 +71,6 @@ class JobHistoryLabelsRule(hitObserver: Observer) Constants.USER_LABEL_TENANT.getValue, classOf[java.util.Map[String, String]] ) - // 当任务的creator是qualitis（或dops）时，tenant不是qualitis发出告警 val listIterator = configMap.keySet.iterator while ({ listIterator.hasNext @@ -82,7 +83,6 @@ class JobHistoryLabelsRule(hitObserver: Observer) } } } - // 当任务代理tenant:Qualitis标签，但是creator不是qualitis标签也进行告警 if (configMap.values().contains(tenant)) { val bimap: HashBiMap[String, String] = HashBiMap.create(configMap) val key = bimap.inverse().get(tenant) @@ -103,9 +103,9 @@ class JobHistoryLabelsRule(hitObserver: Observer) logger.info("hit " + alertData.size() + " data in one iteration") if (alertData.size() > 0) { getHitEvent.notifyObserver(getHitEvent, alertData) - true + true } else { - false + false } } diff --git a/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/app/jobhistory/runtime/CommonJobRunTimeRule.scala b/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/app/jobhistory/runtime/CommonJobRunTimeRule.scala index 1912acf8b4..3c42600432 100644 --- a/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/app/jobhistory/runtime/CommonJobRunTimeRule.scala +++ b/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/app/jobhistory/runtime/CommonJobRunTimeRule.scala @@ -17,24 +17,26 @@ package org.apache.linkis.monitor.scan.app.jobhistory.runtime -import org.apache.commons.lang3.StringUtils import org.apache.linkis.common.utils.Logging import org.apache.linkis.monitor.scan.app.jobhistory.entity.JobHistory import org.apache.linkis.monitor.scan.constants.Constants import org.apache.linkis.monitor.scan.core.ob.Observer import org.apache.linkis.monitor.scan.core.pac.{AbstractScanRule, ScannedData} +import org.apache.commons.lang3.StringUtils + import java.util + import scala.collection.JavaConverters._ /** - * 对前20分钟内的执行数据进行扫描， - * 1.数据的ObserveInfo字段进行判断是否为空， - * 2.任务状态已经完成（Succeed,Failed,Cancelled,Timeout,ALL） - * 满足条件即可触发告警 + * Scan the execution data within the first 20 minutes, + * 1. The ObserveInfo field of the data is judged whether it is empty, 2. The task status has been + * completed (Succeed, Failed, Cancelled, Timeout, ALL) Alarms can be triggered when conditions + * are met */ class CommonJobRunTimeRule(hitObserver: Observer) - extends AbstractScanRule(event = new JobHistoryRunTimeHitEvent, observer = hitObserver) + extends AbstractScanRule(event = new JobHistoryRunTimeHitEvent, observer = hitObserver) with Logging { /** @@ -54,8 +56,12 @@ class CommonJobRunTimeRule(hitObserver: Observer) for (d <- sd.getData().asScala) { d match { case jobHistory: JobHistory => - if (Constants.DIRTY_DATA_FINISHED_JOB_STATUS.contains(jobHistory.getStatus.toUpperCase()) - &&StringUtils.isNotBlank(jobHistory.getObserveInfo)) { + if ( + Constants.DIRTY_DATA_FINISHED_JOB_STATUS.contains( + jobHistory.getStatus.toUpperCase() + ) + && StringUtils.isNotBlank(jobHistory.getObserveInfo) + ) { alertData.add(jobHistory) } else { logger.warn("jobHistory is not completely ， taskid :" + d) @@ -70,9 +76,9 @@ class CommonJobRunTimeRule(hitObserver: Observer) logger.info("hit " + alertData.size() + " data in one iteration") if (alertData.size() > 0) { getHitEvent.notifyObserver(getHitEvent, alertData) - true + true } else { - false + false } } diff --git a/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/app/jobhistory/runtime/CommonRunTimeAlertSender.scala b/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/app/jobhistory/runtime/CommonRunTimeAlertSender.scala index 6aca4c38ff..841c430504 100644 --- a/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/app/jobhistory/runtime/CommonRunTimeAlertSender.scala +++ b/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/app/jobhistory/runtime/CommonRunTimeAlertSender.scala @@ -17,12 +17,6 @@ package org.apache.linkis.monitor.scan.app.jobhistory.runtime -import java.net.InetAddress -import java.text.SimpleDateFormat -import java.util -import java.util.Date - -import org.apache.commons.collections.MapUtils import org.apache.linkis.common.utils.Logging import org.apache.linkis.monitor.scan.app.jobhistory.entity.JobHistory import org.apache.linkis.monitor.scan.app.jobhistory.exception.AnomalyScannerException @@ -31,12 +25,16 @@ import org.apache.linkis.monitor.scan.core.ob.{Event, Observer} import org.apache.linkis.monitor.scan.utils.alert.ims.{MonitorAlertUtils, PooledImsAlertUtils} import org.apache.linkis.server.BDPJettyServerHelper -import scala.collection.JavaConverters._ +import org.apache.commons.collections.MapUtils +import java.net.InetAddress +import java.text.SimpleDateFormat +import java.util +import java.util.Date -class CommonRunTimeAlertSender() - extends Observer - with Logging { +import scala.collection.JavaConverters._ + +class CommonRunTimeAlertSender() extends Observer with Logging { private val dateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss") override def update(e: Event, jobHistoryList: scala.Any): Unit = { @@ -59,10 +57,21 @@ class CommonRunTimeAlertSender() logger.warn("Ignore wrong input data Type : " + a.getClass.getCanonicalName) } else { val jobHistory = a.asInstanceOf[JobHistory] - val observeInfoMap = BDPJettyServerHelper.gson.fromJson(jobHistory.getObserveInfo, classOf[java.util.Map[String, String]]) + val observeInfoMap = BDPJettyServerHelper.gson.fromJson( + jobHistory.getObserveInfo, + classOf[java.util.Map[String, String]] + ) val extraMap = MapUtils.getMap(observeInfoMap, "extra") - observeInfoMap.put("title", extraMap.get("title").toString + ",任务id：" + jobHistory.getId + ",执行结果 :" + jobHistory.getStatus) - observeInfoMap.put("$detail", extraMap.get("detail").toString + ",执行结果 :" + jobHistory.getStatus) + observeInfoMap.put( + "title", + extraMap + .get("title") + .toString + ",任务id：" + jobHistory.getId + ",执行结果 :" + jobHistory.getStatus + ) + observeInfoMap.put( + "$detail", + extraMap.get("detail").toString + ",执行结果 :" + jobHistory.getStatus + ) observeInfoMap.put("$submitUser", jobHistory.getSubmitUser) observeInfoMap.put("$status", jobHistory.getStatus) observeInfoMap.put("$id", jobHistory.getId.toString) @@ -80,9 +89,10 @@ class CommonRunTimeAlertSender() observeInfoMap.put("$ip", InetAddress.getLocalHost.getHostAddress) observeInfoMap.remove("taskId") observeInfoMap.remove("extra") - val alters = MonitorAlertUtils.getAlertsByDss(Constants.JOB_RESULT_IM, observeInfoMap) + val alters = MonitorAlertUtils.getAlerts(Constants.JOB_RESULT_IM, observeInfoMap) PooledImsAlertUtils.addAlert(alters.get("12016")) } } } + } diff --git a/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/app/jobhistory/runtime/JobHistoryRunTimeAlertSender.scala b/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/app/jobhistory/runtime/JobHistoryRunTimeAlertSender.scala index b9f35ce7c7..621aacb2d9 100644 --- a/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/app/jobhistory/runtime/JobHistoryRunTimeAlertSender.scala +++ b/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/app/jobhistory/runtime/JobHistoryRunTimeAlertSender.scala @@ -17,8 +17,6 @@ package org.apache.linkis.monitor.scan.app.jobhistory.runtime -import java.util - import org.apache.linkis.common.utils.Logging import org.apache.linkis.monitor.scan.app.jobhistory.entity.JobHistory import org.apache.linkis.monitor.scan.app.jobhistory.exception.AnomalyScannerException @@ -26,18 +24,18 @@ import org.apache.linkis.monitor.scan.constants.Constants import org.apache.linkis.monitor.scan.core.ob.{Event, Observer} import org.apache.linkis.monitor.scan.utils.alert.ims.{MonitorAlertUtils, PooledImsAlertUtils} +import java.util + import scala.collection.JavaConverters._ /** - * 对前20分钟内的执行数据进行扫描，对已结束的任务进行判断， - * 1.jobhistory中的parm字段中包含（task.notification.conditions） - * 2.执行任务的结果是（Succeed,Failed,Cancelled,Timeout,ALL）其中任意一个，则触发告警 - * 3.job的结果是已经结束 - * 同时满足上述三个条件即可触发告警 + * Scan the execution data within the first 20 minutes, judge the completed tasks, + * 1. The parm field in jobhistory contains (task.notification.conditions) 2. If the result of + * executing the task is any one of (Succeed, Failed, Canceled, Timeout, ALL), an alarm will be + * triggered 3.The result of the job is that it has ended The alarm can be triggered if the + * above three conditions are met at the same time */ -class JobHistoryRunTimeAlertSender() - extends Observer - with Logging { +class JobHistoryRunTimeAlertSender() extends Observer with Logging { override def update(e: Event, jobHistroyList: scala.Any): Unit = { if (!e.isInstanceOf[JobHistoryRunTimeHitEvent]) { @@ -70,4 +68,5 @@ class JobHistoryRunTimeAlertSender() } } } + } diff --git a/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/app/jobhistory/runtime/JobHistoryRunTimeRule.scala b/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/app/jobhistory/runtime/JobHistoryRunTimeRule.scala index e136ea46d0..4f91be337a 100644 --- a/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/app/jobhistory/runtime/JobHistoryRunTimeRule.scala +++ b/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/app/jobhistory/runtime/JobHistoryRunTimeRule.scala @@ -27,11 +27,11 @@ import org.apache.linkis.protocol.utils.TaskUtils import org.apache.linkis.server.BDPJettyServerHelper import java.util -import scala.collection.JavaConverters._ +import scala.collection.JavaConverters._ class JobHistoryRunTimeRule(hitObserver: Observer) - extends AbstractScanRule(event = new JobHistoryRunTimeHitEvent, observer = hitObserver) + extends AbstractScanRule(event = new JobHistoryRunTimeHitEvent, observer = hitObserver) with Logging { private val scanRuleList = CacheUtils.cacheBuilder @@ -52,15 +52,28 @@ class JobHistoryRunTimeRule(hitObserver: Observer) for (d <- sd.getData().asScala) { d match { case jobHistory: JobHistory => - if (Constants.DIRTY_DATA_FINISHED_JOB_STATUS.contains(jobHistory.getStatus.toUpperCase())) { - val parmsMap: util.Map[String, scala.AnyRef] = BDPJettyServerHelper.gson.fromJson(jobHistory.getParams, classOf[util.Map[String, scala.AnyRef]]) + if ( + Constants.DIRTY_DATA_FINISHED_JOB_STATUS.contains( + jobHistory.getStatus.toUpperCase() + ) + ) { + val parmsMap: util.Map[String, scala.AnyRef] = BDPJettyServerHelper.gson.fromJson( + jobHistory.getParams, + classOf[util.Map[String, scala.AnyRef]] + ) val runtimeMap = TaskUtils.getRuntimeMap(parmsMap) - if (runtimeMap.containsKey("task.notification.conditions") && - Constants.DIRTY_DATA_FINISHED_JOB_STATUS.contains(String.valueOf(runtimeMap.get("task.notification.conditions")).toUpperCase())) { - alertData.add(jobHistory) + if ( + runtimeMap.containsKey("task.notification.conditions") && + Constants.DIRTY_DATA_FINISHED_JOB_STATUS.contains( + String.valueOf(runtimeMap.get("task.notification.conditions")).toUpperCase() + ) + ) { + alertData.add(jobHistory) } } else { - logger.warn("Ignored wrong input data Type : " + d + ", " + d.getClass.getCanonicalName) + logger.warn( + "Ignored wrong input data Type : " + d + ", " + d.getClass.getCanonicalName + ) } scanRuleList.put("jobHistoryId", jobHistory.getId) case _ => @@ -73,9 +86,9 @@ class JobHistoryRunTimeRule(hitObserver: Observer) logger.info("hit " + alertData.size() + " data in one iteration") if (alertData.size() > 0) { getHitEvent.notifyObserver(getHitEvent, alertData) - true + true } else { - false + false } } diff --git a/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/client/MonitorHTTPClient.scala b/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/client/MonitorHTTPClient.scala index 8813aba210..97cd429887 100644 --- a/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/client/MonitorHTTPClient.scala +++ b/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/client/MonitorHTTPClient.scala @@ -21,7 +21,11 @@ import org.apache.linkis.httpclient.authentication.AuthenticationStrategy import org.apache.linkis.httpclient.dws.authentication.StaticAuthenticationStrategy import org.apache.linkis.httpclient.dws.config.{DWSClientConfig, DWSClientConfigBuilder} import org.apache.linkis.httpclient.response.Result -import org.apache.linkis.monitor.scan.request.{EmsListAction, EntranceTaskAction, MonitorResourceAction} +import org.apache.linkis.monitor.scan.request.{ + EmsListAction, + EntranceTaskAction, + MonitorResourceAction +} import org.apache.linkis.monitor.scan.response.EntranceTaskResult import org.apache.linkis.ujes.client.response.EmsListResult diff --git a/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/constants/Constants.scala b/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/constants/Constants.scala index f3e6d74c48..89ab330b4d 100644 --- a/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/constants/Constants.scala +++ b/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/constants/Constants.scala @@ -19,76 +19,58 @@ package org.apache.linkis.monitor.scan.constants import org.apache.linkis.common.conf.CommonVars - object Constants { - val ALERT_IMS_URL = CommonVars.properties.getProperty( - "wds.linkis.alert.url", - "http://127.0.0.1:10812/ims_data_access/send_alarm.do" - ) + val SCAN_PREFIX_ERRORCODE = "jobhistory.errorcode." + val SCAN_PREFIX_UNFINISHED_JOBTIME_EXCEED_SEC = "jobhistory.unfinished.time.exceed.sec." + val ALERT_RESOURCE_MONITOR = "ecm.resource.monitor.im." val ALERT_PROPS_FILE_PATH = CommonVars.properties.getProperty( - "wds.linkis.alert.ims.file.path", - "linkis-et-monitor-ims.properties" + "linkis.alert.conf.file.path", + "linkis-et-monitor-file.properties" ) - val ALERT_IMS_MAX_LINES = CommonVars[Int]("wds.linkis.alert.ims.max.lines", 8).getValue - - val SCAN_INTERVALS_SECONDS = - CommonVars[Long]("wds.linkis.errorcode.scanner.interval.seconds", 1 * 60 * 60).getValue - - val MAX_INTERVALS_SECONDS = - CommonVars[Long]("wds.linkis.errorcode.scanner.max.interval.seconds", 1 * 60 * 60).getValue + val ALERT_IMS_URL = CommonVars.properties.getProperty( + "linkis.alert.url", + "http://127.0.0.1:10812/ims_data_access/send_alarm.do" + ) val ALERT_SUB_SYSTEM_ID = - CommonVars.properties.getProperty("wds.linkis.alert.ims.sub_system_id", "5435") + CommonVars.properties.getProperty("linkis.alert.sub_system_id", "10001") val ALERT_DEFAULT_RECEIVERS = CommonVars.properties - .getProperty("wds.linkis.alert.receiver.default", "") + .getProperty("linkis.alert.receiver.default", "") .split(",") .toSet[String] - val SCAN_PREFIX_ERRORCODE = "jobhistory.errorcode." - val SCAN_PREFIX_UNFINISHED_JOBTIME_EXCEED_SEC = "jobhistory.unfinished.time.exceed.sec." - - val SCAN_RULE_UNFINISHED_JOB_STATUS = - "Inited,WaitForRetry,Scheduled,Running".split(",").map(s => s.toUpperCase()) + val ALERT_IMS_MAX_LINES = CommonVars[Int]("linkis.alert.content.max.lines", 8).getValue - val DIRTY_DATA_EUREKA_DELETE_INSTANCE_URL = - CommonVars.apply("wds.linkis.eureka.defaultZone", "http://localhost:20303").getValue + val ERRORCODE_SCAN_INTERVALS_SECONDS = + CommonVars[Long]("linkis.errorcode.scanner.interval.seconds", 1 * 60 * 60).getValue - val DIRTY_DATA_EUREKA_DELETE_PATH = CommonVars - .apply("wds.linkis.dirty.data.eureka.delete.path", "/apps/{springName}/{instance}") - .getValue + val ERRORCODE_MAX_INTERVALS_SECONDS = + CommonVars[Long]("linkis.errorcode.scanner.max.interval.seconds", 1 * 60 * 60).getValue - val DIRTY_DATA_UNFINISHED_JOB_STATUS = + val SCAN_RULE_UNFINISHED_JOB_STATUS = "Inited,WaitForRetry,Scheduled,Running".split(",").map(s => s.toUpperCase()) - val DIRTY_DATA_JOB_TARGET_STATUS = "Cancelled" - - val DIRTY_DATA_ENTRANCE_APPLICATIONNAME = - CommonVars("wds.linkis.entrance.spring.name", "linkis-cg-entrance").getValue - - val MODIFY_DB_DATA_DAYS = CommonVars("wds.linkis.dirty.data.modify.db.days", 1).getValue - val ALERT_RESOURCE_MONITOR = "ecm.resource.monitor.im." - val LINKIS_API_VERSION: CommonVars[String] = - CommonVars[String]("wds.linkis.bml.api.version", "v1") + CommonVars[String]("linkis.bml.api.version", "v1") val AUTH_TOKEN_KEY: CommonVars[String] = - CommonVars[String]("wds.linkis.bml.auth.token.key", "Validation-Code") + CommonVars[String]("linkis.bml.auth.token.key", "Validation-Code") val AUTH_TOKEN_VALUE: CommonVars[String] = - CommonVars[String]("wds.linkis.bml.auth.token.value", "BML-AUTH") + CommonVars[String]("linkis.bml.auth.token.value", "BML-AUTH") val CONNECTION_MAX_SIZE: CommonVars[Int] = - CommonVars[Int]("wds.linkis.bml.connection.max.size", 10) + CommonVars[Int]("linkis.bml.connection.max.size", 10) val CONNECTION_TIMEOUT: CommonVars[Int] = - CommonVars[Int]("wds.linkis.bml.connection.timeout", 5 * 60 * 1000) + CommonVars[Int]("linkis.bml.connection.timeout", 5 * 60 * 1000) val CONNECTION_READ_TIMEOUT: CommonVars[Int] = - CommonVars[Int]("wds.linkis.bml.connection.read.timeout", 10 * 60 * 1000) + CommonVars[Int]("linkis.bml.connection.read.timeout", 10 * 60 * 1000) val AUTH_TOKEN_KEY_SHORT_NAME = "tokenKey" val AUTH_TOKEN_VALUE_SHORT_NAME = "tokenValue" @@ -106,8 +88,4 @@ object Constants { val THREAD_TIME_OUT_IM = "thread.monitor.timeout.im." val JOB_RESULT_IM = "jobhistory.result.monitor.im." - val DIRTY_DATA_FINISHED_JOB_STATUS = - "Succeed,Failed,Cancelled,Timeout,ALL".split(",").map(s => s.toUpperCase()) - val DIRTY_DATA_FINISHED_JOB_STATUS_ARRAY = "Succeed,Failed,Cancelled,Timeout".split(",") - } diff --git a/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/constants/ScanOperatorEnum.scala b/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/constants/ScanOperatorEnum.scala index 8ff3755747..05b244a458 100644 --- a/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/constants/ScanOperatorEnum.scala +++ b/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/constants/ScanOperatorEnum.scala @@ -17,7 +17,6 @@ package org.apache.linkis.monitor.scan.constants - object ScanOperatorEnum extends Enumeration { type ScanOperatorEnum = Value val BML_VERSION, JOB_HISTORY = Value diff --git a/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/core/ob/Event.scala b/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/core/ob/Event.scala index 60322c5814..888a3aa20b 100644 --- a/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/core/ob/Event.scala +++ b/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/core/ob/Event.scala @@ -17,7 +17,6 @@ package org.apache.linkis.monitor.scan.core.ob - trait Event { def isRegistered: Boolean diff --git a/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/core/ob/Observer.scala b/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/core/ob/Observer.scala index 58849dd06c..04359e309f 100644 --- a/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/core/ob/Observer.scala +++ b/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/core/ob/Observer.scala @@ -17,7 +17,6 @@ package org.apache.linkis.monitor.scan.core.ob - trait Observer { /** diff --git a/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/core/pac/AbstractDataFetcher.scala b/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/core/pac/AbstractDataFetcher.scala index 02ad320c55..2f7f9a67e6 100644 --- a/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/core/pac/AbstractDataFetcher.scala +++ b/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/core/pac/AbstractDataFetcher.scala @@ -17,7 +17,6 @@ package org.apache.linkis.monitor.scan.core.pac - abstract class AbstractDataFetcher(customName: String = "") extends DataFetcher { private val name: String = if (!customName.isEmpty) { diff --git a/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/core/pac/AbstractScanRule.scala b/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/core/pac/AbstractScanRule.scala index 8d6762c562..f572bd90bd 100644 --- a/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/core/pac/AbstractScanRule.scala +++ b/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/core/pac/AbstractScanRule.scala @@ -21,7 +21,6 @@ import org.apache.linkis.common.utils.Logging import org.apache.linkis.monitor.scan.core.ob.{Event, Observer} import org.apache.linkis.monitor.scan.core.ob.Observer - abstract class AbstractScanRule(customName: String = "", event: Event, observer: Observer) extends ScanRule with Logging { diff --git a/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/core/pac/BaseScannedData.scala b/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/core/pac/BaseScannedData.scala index 02f32a3fc5..efc573132a 100644 --- a/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/core/pac/BaseScannedData.scala +++ b/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/core/pac/BaseScannedData.scala @@ -19,7 +19,6 @@ package org.apache.linkis.monitor.scan.core.pac import java.util - class BaseScannedData(owner: String, data: util.List[scala.Any]) extends ScannedData { override def getOwner(): String = this.owner diff --git a/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/core/pac/ScanBuffer.scala b/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/core/pac/ScanBuffer.scala index d38c02dea7..8075792161 100644 --- a/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/core/pac/ScanBuffer.scala +++ b/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/core/pac/ScanBuffer.scala @@ -20,7 +20,6 @@ package org.apache.linkis.monitor.scan.core.pac import java.util import java.util.concurrent.LinkedBlockingDeque - class ScanBuffer { val buffer: LinkedBlockingDeque[ScannedData] = new LinkedBlockingDeque[ScannedData] diff --git a/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/core/pac/ScannedData.scala b/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/core/pac/ScannedData.scala index a6914c3e60..2f5cc68208 100644 --- a/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/core/pac/ScannedData.scala +++ b/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/core/pac/ScannedData.scala @@ -19,7 +19,6 @@ package org.apache.linkis.monitor.scan.core.pac import java.util - trait ScannedData { def getOwner(): String diff --git a/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/core/scanner/AbstractScanner.scala b/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/core/scanner/AbstractScanner.scala index e6939df36e..17fae63fb0 100644 --- a/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/core/scanner/AbstractScanner.scala +++ b/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/core/scanner/AbstractScanner.scala @@ -17,15 +17,14 @@ package org.apache.linkis.monitor.scan.core.scanner -import java.util -import java.util.concurrent.CopyOnWriteArrayList -import java.util.concurrent.atomic.AtomicInteger - import org.apache.linkis.common.utils.Logging import org.apache.linkis.monitor.scan.app.jobhistory.exception.AnomalyScannerException import org.apache.linkis.monitor.scan.core.ob.{Event, Observer} import org.apache.linkis.monitor.scan.core.pac._ +import java.util +import java.util.concurrent.CopyOnWriteArrayList +import java.util.concurrent.atomic.AtomicInteger abstract class AbstractScanner extends AnomalyScanner with Logging { private val buffer: ScanBuffer = new ScanBuffer @@ -71,8 +70,8 @@ abstract class AbstractScanner extends AnomalyScanner with Logging { } /** - * Returns a buffer that allows read/write simultaneously buffer is allowed to be written by - * other thread + * Returns a buffer that allows read/write simultaneously buffer is allowed to be written by other + * thread */ override def getBuffer(): ScanBuffer = buffer @@ -102,10 +101,7 @@ abstract class AbstractScanner extends AnomalyScanner with Logging { */ override def run(): Unit = { if (dataFetcherList.size() == 0) { - throw new AnomalyScannerException( - 21304, - "attempting to run scanner with empty dataFetchers" - ) + throw new AnomalyScannerException(21304, "attempting to run scanner with empty dataFetchers") } if (buffer == null) { throw new AnomalyScannerException(21304, "attempting to run scanner with null buffer") @@ -138,8 +134,8 @@ abstract class AbstractScanner extends AnomalyScanner with Logging { } /** - * 1. should be a blocking call 2. read from [[ScanBuffer]] 2. see if [[ScanRule]] is matched - * 3. trigger [[Observer]] + * 1. should be a blocking call 2. read from [[ScanBuffer]] 2. see if [[ScanRule]] is matched 3. + * trigger [[Observer]] */ override def analyzeOneIteration(): Unit = { val dataToAnalyze = buffer.drain() @@ -158,8 +154,8 @@ abstract class AbstractScanner extends AnomalyScanner with Logging { } /** - * 1. should be non-blocking 2. keeps calling scanOneIteration() and analyzeOneIteration() - * utils stop() is called + * 1. should be non-blocking 2. keeps calling scanOneIteration() and analyzeOneIteration() utils + * stop() is called */ override def start(): Unit = { // TODO diff --git a/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/core/scanner/AnomalyScanner.scala b/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/core/scanner/AnomalyScanner.scala index 854ca78c3f..d887e7bf86 100644 --- a/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/core/scanner/AnomalyScanner.scala +++ b/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/core/scanner/AnomalyScanner.scala @@ -81,14 +81,14 @@ trait AnomalyScanner { protected def scanOneIteration(): Unit /** - * Returns a buffer that allows read/write simultaneously buffer is allowed to be written by - * other thread + * Returns a buffer that allows read/write simultaneously buffer is allowed to be written by other + * thread */ protected def getBuffer(): ScanBuffer /** - * 1. should be a blocking call 2. read from [[ScanBuffer]] 2. see if [[ScanRule]] is matched - * 3. trigger [[[[org.apache.linkis.tools.core.ob.Observer]]]] + * 1. should be a blocking call 2. read from [[ScanBuffer]] 2. see if [[ScanRule]] is matched 3. + * trigger [[[[org.apache.linkis.tools.core.ob.Observer]]]] */ protected def analyzeOneIteration(): Unit diff --git a/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/core/scanner/DefaultScanner.scala b/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/core/scanner/DefaultScanner.scala index c56a3f8182..2463439873 100644 --- a/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/core/scanner/DefaultScanner.scala +++ b/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/core/scanner/DefaultScanner.scala @@ -19,7 +19,6 @@ package org.apache.linkis.monitor.scan.core.scanner import org.apache.linkis.monitor.scan.utils.alert.ims.PooledImsAlertUtils - class DefaultScanner extends AbstractScanner { override def shutdown(): Unit = { diff --git a/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/request/EntranceTaskAction.scala b/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/request/EntranceTaskAction.scala index 6126a2b04b..94636a41ca 100644 --- a/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/request/EntranceTaskAction.scala +++ b/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/request/EntranceTaskAction.scala @@ -57,7 +57,8 @@ object EntranceTaskAction { def build(): EntranceTaskAction = { val entranceTaskAction = new EntranceTaskAction if (StringUtils.isNotBlank(creator)) entranceTaskAction.setParameter("creator", creator) - if (StringUtils.isNotBlank(engineTypeLabel)) entranceTaskAction.setParameter("engineTypeLabel", engineTypeLabel) + if (StringUtils.isNotBlank(engineTypeLabel)) + entranceTaskAction.setParameter("engineTypeLabel", engineTypeLabel) if (StringUtils.isNotBlank(instance)) entranceTaskAction.setParameter("instance", instance) if (StringUtils.isNotBlank(user)) { // hadoop用户应该获取全部用户entrance信息，则无需传user，即可获取全部entrance信息 diff --git a/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/utils/alert/AlertDesc.scala b/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/utils/alert/AlertDesc.scala index b1f29530f8..068746667c 100644 --- a/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/utils/alert/AlertDesc.scala +++ b/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/utils/alert/AlertDesc.scala @@ -17,7 +17,6 @@ package org.apache.linkis.monitor.scan.utils.alert - trait AlertDesc { /** diff --git a/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/utils/alert/AlertSender.scala b/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/utils/alert/AlertSender.scala index 5ae1b960e5..fb2cff458d 100644 --- a/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/utils/alert/AlertSender.scala +++ b/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/utils/alert/AlertSender.scala @@ -17,7 +17,6 @@ package org.apache.linkis.monitor.scan.utils.alert - trait AlertSender { /** diff --git a/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/utils/alert/PooledAlertSender.scala b/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/utils/alert/PooledAlertSender.scala index 10eb367d2a..5ff274438a 100644 --- a/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/utils/alert/PooledAlertSender.scala +++ b/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/utils/alert/PooledAlertSender.scala @@ -23,9 +23,8 @@ import org.apache.linkis.common.utils.{Logging, Utils} import java.util.concurrent.{Future, LinkedBlockingQueue} import java.util.concurrent.atomic.{AtomicBoolean, AtomicInteger} - abstract class PooledAlertSender extends AlertSender with Logging { - private val THREAD_POOL_SIZE = CommonVars[Int]("wds.linkis.alert.pool.size", 5).getValue + private val THREAD_POOL_SIZE = CommonVars[Int]("linkis.alert.pool.size", 5).getValue private val alertDescQ: LinkedBlockingQueue[AlertDesc] = new LinkedBlockingQueue[AlertDesc](1000) @@ -58,15 +57,15 @@ abstract class PooledAlertSender extends AlertSender with Logging { def start(): Unit = { future = Utils.defaultScheduler.submit(new Runnable() { override def run() { - info("Pooled alert thread started!") + logger.info("Pooled alert thread started!") while (!stopped.get) { executors synchronized { while (!stopped.get && runningNumber.get >= THREAD_POOL_SIZE) { - info("Pooled alert thread is full, start waiting") + logger.info("Pooled alert thread is full, start waiting") executors.wait() } } - info("Pooled alert thread continue processing") + logger.info("Pooled alert thread continue processing") if (stopped.get && alertDescQ.size() == 0) return val alertDesc = Utils.tryQuietly(alertDescQ.take) @@ -75,12 +74,12 @@ abstract class PooledAlertSender extends AlertSender with Logging { override def run() { runningNumber.addAndGet(1) Utils.tryAndWarn { - info("sending alert , information: " + alertDesc) + logger.info("sending alert , information: " + alertDesc) val ok = doSendAlert(alertDesc) if (!ok) { warn("Failed to send alert: " + alertDesc) } else { - info("successfully send alert: " + alertDesc) + logger.info("successfully send alert: " + alertDesc) } runningNumber.decrementAndGet executors synchronized executors.notify @@ -93,7 +92,7 @@ abstract class PooledAlertSender extends AlertSender with Logging { } def shutdown(waitComplete: Boolean = true, timeoutMs: Long = -1): Unit = { - info("stopping the Pooled alert thread...") + logger.info("stopping the Pooled alert thread...") if (waitComplete) { val startTime = System.currentTimeMillis() while ( @@ -106,7 +105,7 @@ abstract class PooledAlertSender extends AlertSender with Logging { executors.shutdown stopped.set(true) future.cancel(true) - info("Pooled alert thread is stopped") + logger.info("Pooled alert thread is stopped") } } diff --git a/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/utils/alert/ims/ImsAlertDesc.scala b/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/utils/alert/ims/ImsAlertDesc.scala index 6b12da9584..6aa19d5092 100644 --- a/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/utils/alert/ims/ImsAlertDesc.scala +++ b/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/utils/alert/ims/ImsAlertDesc.scala @@ -32,7 +32,6 @@ import scala.collection.JavaConverters._ import ImsAlertLevel.ImsAlertLevel import ImsAlertWay.ImsAlertWay - case class ImsAlertDesc( var subSystemId: String, var alertTitle: String, @@ -149,7 +148,7 @@ case class ImsAlertDesc( alertReceivers } - Array(subSystemId, newAlertTitle, newAlertObj, newAlertInfo, newAlertReceivers) + Array(subSystemId, newAlertTitle, newAlertObj, newAlertInfo, newAlertReceivers) } } diff --git a/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/utils/alert/ims/ImsAlertLevel.scala b/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/utils/alert/ims/ImsAlertLevel.scala index cb304e9e4f..13948eac82 100644 --- a/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/utils/alert/ims/ImsAlertLevel.scala +++ b/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/utils/alert/ims/ImsAlertLevel.scala @@ -17,7 +17,6 @@ package org.apache.linkis.monitor.scan.utils.alert.ims - object ImsAlertLevel extends Enumeration { type ImsAlertLevel = Value val INFO = Value("5") diff --git a/linkis-extensions/linkis-et-monitor/src/test/java/org/apache/linkis/monitor/scan/app/LinkisJobHistoryScanApplicationTest.java b/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/utils/alert/ims/ImsAlertPropFileData.scala similarity index 52% rename from linkis-extensions/linkis-et-monitor/src/test/java/org/apache/linkis/monitor/scan/app/LinkisJobHistoryScanApplicationTest.java rename to linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/utils/alert/ims/ImsAlertPropFileData.scala index 716ad5cf43..d5fb95e910 100644 --- a/linkis-extensions/linkis-et-monitor/src/test/java/org/apache/linkis/monitor/scan/app/LinkisJobHistoryScanApplicationTest.java +++ b/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/utils/alert/ims/ImsAlertPropFileData.scala @@ -15,24 +15,16 @@ * limitations under the License. */ -package org.apache.linkis.monitor.scan.app; +package org.apache.linkis.monitor.scan.utils.alert.ims -import org.apache.linkis.server.utils.LinkisMainHelper; +import com.fasterxml.jackson.annotation.JsonProperty - -public class LinkisJobHistoryScanApplicationTest { - // @Before - public void before() { - System.getProperties().setProperty(LinkisMainHelper.SERVER_NAME_KEY(), "linkis-et-monitor"); - System.getProperties() - .setProperty("log4j.configurationFile", "src/test/resources/log4j2-console.xml"); - // System.getProperties().setProperty("wds.linkis.server.conf", - // "linkis-et-monitor.properties"); - } - - // @Test - public void main() throws Exception { - LinkisJobHistoryScanApplication.main(new String[] {}); - // LinkisJobHistoryScanApplication.main(new String[]{"2021122919", "2021122921"}); - } -} +case class ImsAlertPropFileData( + @JsonProperty("alert_title") alertTitle: String, + @JsonProperty("alert_info") alertInfo: String, + @JsonProperty("alert_way") alertWays: String, + @JsonProperty("alert_reciver") alertReceivers: String, + @JsonProperty("alert_level") alertLevel: String, + @JsonProperty("alert_obj") alertObj: String, + @JsonProperty("can_recover") canRecover: String +) diff --git a/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/utils/alert/ims/ImsAlertWay.scala b/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/utils/alert/ims/ImsAlertWay.scala index 207df613d7..244e995775 100644 --- a/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/utils/alert/ims/ImsAlertWay.scala +++ b/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/utils/alert/ims/ImsAlertWay.scala @@ -17,11 +17,10 @@ package org.apache.linkis.monitor.scan.utils.alert.ims - object ImsAlertWay extends Enumeration { type ImsAlertWay = Value val NoAlert = Value("0") - val RTX = Value("1") + val WXWork = Value("1") val Email = Value("2") val WeChat = Value("3") } diff --git a/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/utils/alert/ims/ImsRequest.scala b/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/utils/alert/ims/ImsRequest.scala index 64b80d3a95..4e93f53a05 100644 --- a/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/utils/alert/ims/ImsRequest.scala +++ b/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/utils/alert/ims/ImsRequest.scala @@ -21,7 +21,6 @@ import java.util import com.fasterxml.jackson.annotation.JsonProperty - case class ImsRequest(@JsonProperty("alertList") alertList: util.List[AlertEntity]) case class AlertEntity( diff --git a/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/utils/alert/ims/JobHistoryScanImsAlertPropFileParserUtils.scala b/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/utils/alert/ims/JobHistoryScanImsAlertPropFileParserUtils.scala deleted file mode 100644 index 95b64f3f4e..0000000000 --- a/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/utils/alert/ims/JobHistoryScanImsAlertPropFileParserUtils.scala +++ /dev/null @@ -1,132 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.monitor.scan.utils.alert.ims - -import org.apache.linkis.common.utils.{JsonUtils, Logging, Utils} -import org.apache.linkis.monitor.scan.app.jobhistory.exception.AnomalyScannerException -import org.apache.linkis.monitor.scan.constants.Constants -import org.apache.linkis.monitor.scan.utils.alert.AlertDesc -import org.apache.commons.lang3.StringUtils -import org.apache.commons.lang3.exception.ExceptionUtils - -import java.io.{BufferedReader, File, FileInputStream, InputStream, InputStreamReader} -import java.text.SimpleDateFormat -import java.util -import java.util.Properties -import scala.collection.JavaConverters._ -import com.fasterxml.jackson.annotation.JsonProperty -import com.fasterxml.jackson.databind.{DeserializationFeature, ObjectMapper} -import com.fasterxml.jackson.module.scala.DefaultScalaModule -import org.apache.commons.io.IOUtils - - -object JobHistoryScanImsAlertPropFileParserUtils extends Logging { - - private val mapper = { - val ret = new ObjectMapper().setDateFormat(new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ssZ")) - ret.registerModule(DefaultScalaModule) - ret.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false) - ret - } - - def getAlerts(prefix: String): util.Map[String, AlertDesc] = { - val ret = new util.HashMap[String, AlertDesc]() - val url = getClass.getClassLoader.getResource(Constants.ALERT_PROPS_FILE_PATH) - if (url == null) { - throw new AnomalyScannerException( - 21304, - "Failed to load alerts from alert properties. Alert properties file does not exist: " + Constants.ALERT_PROPS_FILE_PATH - ) - } - logger.info("reading alert properties from: " + url.getFile) - val properties = new Properties() - var inputStream: InputStream = null - var reader: InputStreamReader = null - var buff: BufferedReader = null - - Utils.tryFinally { - Utils.tryCatch { - inputStream = new FileInputStream(new File(url.getFile)) - reader = new InputStreamReader(inputStream, "UTF-8") - buff = new BufferedReader(reader) - properties.load(buff) - } { t => { - throw new AnomalyScannerException(21304, "Failed to load alerts from alert properties. Cause: " + ExceptionUtils.getMessage(t)) - return ret - } - } - } { - IOUtils.closeQuietly(buff) - IOUtils.closeQuietly(reader) - IOUtils.closeQuietly(inputStream) - } - for ((k: String, v: String) <- properties.asScala) { - if (ret.containsKey(k)) { - logger.warn("found duplicate key in alert properties, accept only the first one") - } else if (StringUtils.startsWith(k, prefix)) { - val data = mapper.readValue(v, classOf[ImsAlertPropFileData]) - val receivers = { - val set: util.Set[String] = new util.HashSet[String] - if (StringUtils.isNotBlank(data.alertReceivers)) { - data.alertReceivers.split(",").map(r => set.add(r)) - } - Constants.ALERT_DEFAULT_RECEIVERS.foreach(e => { - if (StringUtils.isNotBlank(e)) { - set.add(e) - } - }) - set - } - val alertDesc = Utils.tryAndWarn( - new ImsAlertDesc( - Constants.ALERT_SUB_SYSTEM_ID, - data.alertTitle, - data.alertObj, - data.alertInfo, - ImsAlertLevel.withName(data.alertLevel), - null, // Not used - 0, { - val set: util.Set[ImsAlertWay.Value] = new util.HashSet[ImsAlertWay.Value] - if (StringUtils.isNotBlank(data.alertWays)) { - data.alertWays - .split(",") - .map(alertWayStr => set.add(ImsAlertWay.withName(alertWayStr))) - } - set - }, - receivers - ) - ) - val realK = StringUtils.substringAfter(k, prefix) - ret.put(realK, alertDesc) - } - } - ret - } - -} - -case class ImsAlertPropFileData( - @JsonProperty("alert_title") alertTitle: String, - @JsonProperty("alert_info") alertInfo: String, - @JsonProperty("alert_way") alertWays: String, - @JsonProperty("alert_reciver") alertReceivers: String, - @JsonProperty("alert_level") alertLevel: String, - @JsonProperty("alert_obj") alertObj: String, - @JsonProperty("can_recover") canRecover: String -) diff --git a/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/utils/alert/ims/MonitorAlertUtils.scala b/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/utils/alert/ims/MonitorAlertUtils.scala index 021ec3b588..285f5075dd 100644 --- a/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/utils/alert/ims/MonitorAlertUtils.scala +++ b/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/utils/alert/ims/MonitorAlertUtils.scala @@ -21,6 +21,8 @@ import org.apache.linkis.common.utils.{JsonUtils, Logging, Utils} import org.apache.linkis.monitor.scan.app.jobhistory.exception.AnomalyScannerException import org.apache.linkis.monitor.scan.constants.Constants import org.apache.linkis.monitor.scan.utils.alert.AlertDesc + +import org.apache.commons.io.IOUtils import org.apache.commons.lang3.StringUtils import org.apache.commons.lang3.exception.ExceptionUtils @@ -28,14 +30,14 @@ import java.io.{BufferedReader, File, FileInputStream, InputStream, InputStreamR import java.text.SimpleDateFormat import java.util import java.util.Properties + import scala.collection.JavaConverters._ + import com.fasterxml.jackson.databind.{DeserializationFeature, ObjectMapper} import com.fasterxml.jackson.module.scala.DefaultScalaModule -import org.apache.commons.io.IOUtils object MonitorAlertUtils extends Logging { - private val mapper = { val ret = new ObjectMapper().setDateFormat(new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ssZ")) ret.registerModule(DefaultScalaModule) @@ -43,11 +45,7 @@ object MonitorAlertUtils extends Logging { ret } - def getAlerts( - prefix: String, - parms: util.HashMap[String, String] - ): util.Map[String, AlertDesc] = { - val ret = new util.HashMap[String, AlertDesc]() + val properties = { val url = getClass.getClassLoader.getResource(Constants.ALERT_PROPS_FILE_PATH) if (url == null) { throw new AnomalyScannerException( @@ -58,29 +56,37 @@ object MonitorAlertUtils extends Logging { logger.info("reading alert properties from: " + url.getFile) val properties = new Properties() var inputStream: InputStream = null - var reader: InputStreamReader = null var buff: BufferedReader = null - Utils.tryFinally { - Utils.tryThrow { - inputStream = new FileInputStream(new File(url.getFile)) - reader = new InputStreamReader(inputStream, "UTF-8") - buff = new BufferedReader(reader) - properties.load(buff) - } { - case t: Throwable => - new AnomalyScannerException( + Utils.tryCatch { + inputStream = new FileInputStream(new File(url.getFile)) + reader = new InputStreamReader(inputStream, "UTF-8") + buff = new BufferedReader(reader) + properties.load(buff) + } { t => + { + throw new AnomalyScannerException( 21304, "Failed to load alerts from alert properties. Cause: " + ExceptionUtils.getMessage(t) ) } + } } { IOUtils.closeQuietly(buff) IOUtils.closeQuietly(reader) IOUtils.closeQuietly(inputStream) } - for ((k: String, v: String) <- properties.asScala) { + properties.asScala + } + + def getAlerts( + prefix: String, + params: util.HashMap[String, String] + ): util.Map[String, AlertDesc] = { + val ret = new util.HashMap[String, AlertDesc]() + + for ((k: String, v: String) <- properties) { if (ret.containsKey(k)) { logger.warn("found duplicate key in alert properties, accept only the first one") } else if (StringUtils.startsWith(k, prefix)) { @@ -89,10 +95,10 @@ object MonitorAlertUtils extends Logging { new StringBuilder().append(data.alertInfo).toString().getBytes(), "utf-8" ).replace("$name", data.alertReceivers) - val interator = parms.keySet.iterator + val interator = params.keySet.iterator while (interator.hasNext) { val key = interator.next - val value = parms.get(key) + val value = params.get(key) alertInfo = alertInfo.replace(key, value) } val receivers = { @@ -100,113 +106,38 @@ object MonitorAlertUtils extends Logging { if (StringUtils.isNotBlank(data.alertReceivers)) { data.alertReceivers.split(",").map(r => set.add(r)) } - Constants.ALERT_DEFAULT_RECEIVERS.foreach(e => { - if (StringUtils.isNotBlank(e)) { - set.add(e) - } - }) - set.add(parms.get("$alteruser")) - set - } - val alertDesc = Utils.tryAndWarn( - new ImsAlertDesc( - Constants.ALERT_SUB_SYSTEM_ID, - data.alertTitle, - data.alertObj, - alertInfo, - ImsAlertLevel.withName(data.alertLevel), - null, // Not used - 0, { - val set: util.Set[ImsAlertWay.Value] = new util.HashSet[ImsAlertWay.Value] - if (StringUtils.isNotBlank(data.alertWays)) { - data.alertWays - .split(",") - .map(alertWayStr => set.add(ImsAlertWay.withName(alertWayStr))) + if (!params.containsKey("$alteruser")) { + Constants.ALERT_DEFAULT_RECEIVERS.foreach(e => { + if (StringUtils.isNotBlank(e)) { + set.add(e) } - set - }, - receivers - ) - ) - val realK = StringUtils.substringAfter(k, prefix) - ret.put(realK, alertDesc) - } - } - ret - } - - - def getAlertsByDss(prefix: String, parms: util.Map[String, String]): util.Map[String, AlertDesc] = { - val ret = new util.HashMap[String, AlertDesc]() - val url = getClass.getClassLoader.getResource(Constants.ALERT_PROPS_FILE_PATH) - if (url == null) { - throw new AnomalyScannerException( - 21304, - "Failed to load alerts from alert properties. Alert properties file does not exist: " + Constants.ALERT_PROPS_FILE_PATH - ) - } - logger.info("reading alert properties from: " + url.getFile) - val properties = new Properties() - var inputStream: InputStream = null - - var reader: InputStreamReader = null - var buff: BufferedReader = null - - Utils.tryFinally { - Utils.tryThrow{ - inputStream = new FileInputStream(new File(url.getFile)) - reader = new InputStreamReader(inputStream, "UTF-8") - buff = new BufferedReader(reader) - properties.load(buff) - } { - case t: Throwable => - new AnomalyScannerException( - 21304, - "Failed to load alerts from alert properties. Cause: " + ExceptionUtils.getMessage(t) - ) - } - } { - IOUtils.closeQuietly(buff) - IOUtils.closeQuietly(reader) - IOUtils.closeQuietly(inputStream) - } - for ((k: String, v: String) <- properties.asScala) { - if (ret.containsKey(k)) { - warn("found duplicate key in alert properties, accept only the first one") - } else if (StringUtils.startsWith(k, prefix)) { - val data = mapper.readValue(v, classOf[ImsAlertPropFileData]) - var alertInfo = data.alertInfo - val interator = parms.keySet.iterator - while (interator.hasNext) { - val key = interator.next - val value = parms.get(key) - alertInfo = alertInfo.replace(key, value) - } -// alertInfo = parms.getOrDefault("detail", "").concat(alertInfo) - val receivers = { - val set: util.Set[String] = new util.HashSet[String] - if (StringUtils.isNotBlank(data.alertReceivers)) { - data.alertReceivers.split(",").map(r => set.add(r)) + }) + } else { + set.add(params.get("$alteruser")) } - Constants.ALERT_DEFAULT_RECEIVERS.foreach(e => { - if (StringUtils.isNotBlank(e)) { - set.add(e) - } - }) - if (StringUtils.isNotBlank(parms.get("receiver"))) { - parms.get("receiver").split(",").map(r => set.add(r)) + if (StringUtils.isNotBlank(params.get("receiver"))) { + params.get("receiver").split(",").map(r => set.add(r)) } set } + + val subSystemId = params.getOrDefault("subSystemId", Constants.ALERT_SUB_SYSTEM_ID) + val alertTitle = params.getOrDefault("title", data.alertTitle) + val alertLevel = + if (StringUtils.isNotBlank(data.alertLevel)) { + ImsAlertLevel.withName(params.getOrDefault("monitorLevel", data.alertLevel)) + } else { + ImsAlertLevel.withName(params.getOrDefault("monitorLevel", ImsAlertLevel.WARN.toString)) + } + val alertDesc = Utils.tryAndWarn( - ImsAlertDesc( - parms.getOrDefault("subSystemId", Constants.ALERT_SUB_SYSTEM_ID), - parms.getOrDefault("title", ""), + ImsAlertDesc( + subSystemId, + alertTitle, data.alertObj, - alertInfo - , - ImsAlertLevel.withName(parms.getOrDefault("monitorLevel", "4")), - null, // Not used + alertInfo, + alertLevel, + null, 0, { val set: util.Set[ImsAlertWay.Value] = new util.HashSet[ImsAlertWay.Value] if (StringUtils.isNotBlank(data.alertWays)) { diff --git a/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/utils/alert/ims/PooledImsAlertSender.scala b/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/utils/alert/ims/PooledImsAlertSender.scala index d2508b37a3..a553cbba89 100644 --- a/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/utils/alert/ims/PooledImsAlertSender.scala +++ b/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/utils/alert/ims/PooledImsAlertSender.scala @@ -34,15 +34,9 @@ import java.util import com.fasterxml.jackson.databind.ObjectMapper import com.fasterxml.jackson.module.scala.DefaultScalaModule +class PooledImsAlertSender(alertUrl: String) extends PooledAlertSender with Logging { -class PooledImsAlertSender( - subSystemId: String, - alertUrl: String, - default_Receivers: util.Set[String] -) extends PooledAlertSender - with Logging { - - protected val httpClient = HttpClients.createDefault // TODO: Linkis-httpclient + protected val httpClient = HttpClients.createDefault private val mapper = new ObjectMapper().setDateFormat(new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ssZ")) @@ -74,7 +68,7 @@ class PooledImsAlertSender( return false } if (paramContent.isEmpty) { - logger. warn("alertParams is empty, will not send alarm") + logger.warn("alertParams is empty, will not send alarm") return false } @@ -99,7 +93,7 @@ class PooledImsAlertSender( LogUtils.stdOutLogger.info("Alert: " + paramContent + "Response: " + responseInfo) if (response.getStatusLine.getStatusCode == 200) return true } - return false + false } override def shutdown(waitComplete: Boolean = true, timeoutMs: Long = -1): Unit = { diff --git a/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/utils/alert/ims/PooledImsAlertUtils.scala b/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/utils/alert/ims/PooledImsAlertUtils.scala index 4a50161438..37fd35724b 100644 --- a/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/utils/alert/ims/PooledImsAlertUtils.scala +++ b/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/utils/alert/ims/PooledImsAlertUtils.scala @@ -33,15 +33,10 @@ import scala.collection.JavaConverters._ import ImsAlertWay.ImsAlertWay - object PooledImsAlertUtils extends Logging { private val sender: PooledImsAlertSender = { - val ret = new PooledImsAlertSender( - Constants.ALERT_SUB_SYSTEM_ID, - Constants.ALERT_IMS_URL, - Constants.ALERT_DEFAULT_RECEIVERS.asJava - ) + val ret = new PooledImsAlertSender(Constants.ALERT_IMS_URL) ret.start() ret } @@ -70,11 +65,11 @@ object PooledImsAlertUtils extends Logging { if (CollectionUtils.isNotEmpty(alertWays)) alertWays else new HashSet[ImsAlertWay]() val (alertInfo, alertLevel) = if (t != null) { _alertWays.add(ImsAlertWay.Email) - _alertWays.add(ImsAlertWay.RTX) + _alertWays.add(ImsAlertWay.WXWork) _alertWays.add(ImsAlertWay.WeChat) (ExceptionUtils.getRootCauseMessage(t), ImsAlertLevel.MAJOR) } else { - _alertWays.add(ImsAlertWay.RTX) + _alertWays.add(ImsAlertWay.WXWork) (message, ImsAlertLevel.WARN) } val alertDesc = new ImsAlertDesc( diff --git a/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/utils/alert/ims/UserLabelAlertUtils.scala b/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/utils/alert/ims/UserLabelAlertUtils.scala index 1083967783..d5fb4a9b07 100644 --- a/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/utils/alert/ims/UserLabelAlertUtils.scala +++ b/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/utils/alert/ims/UserLabelAlertUtils.scala @@ -21,6 +21,8 @@ import org.apache.linkis.common.utils.{JsonUtils, Logging, Utils} import org.apache.linkis.monitor.scan.app.jobhistory.exception.AnomalyScannerException import org.apache.linkis.monitor.scan.constants.Constants import org.apache.linkis.monitor.scan.utils.alert.AlertDesc + +import org.apache.commons.io.IOUtils import org.apache.commons.lang3.StringUtils import org.apache.commons.lang3.exception.ExceptionUtils @@ -28,99 +30,18 @@ import java.io.{BufferedReader, File, FileInputStream, InputStream, InputStreamR import java.text.SimpleDateFormat import java.util import java.util.Properties + import scala.collection.JavaConverters._ + import com.fasterxml.jackson.databind.{DeserializationFeature, ObjectMapper} import com.fasterxml.jackson.module.scala.DefaultScalaModule -import org.apache.commons.io.IOUtils object UserLabelAlertUtils extends Logging { - private val mapper = { - val ret = new ObjectMapper().setDateFormat(new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ssZ")) - ret.registerModule(DefaultScalaModule) - ret.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false) - ret - } - - def getAlerts(prefix: String, instans: String): util.Map[String, AlertDesc] = { - val ret = new util.HashMap[String, AlertDesc]() - val url = getClass.getClassLoader.getResource(Constants.ALERT_PROPS_FILE_PATH) - if (url == null) { - throw new AnomalyScannerException( - 21304, - "Failed to load alerts from alert properties. Alert properties file does not exist: " + Constants.ALERT_PROPS_FILE_PATH - ) - } - logger.info("reading alert properties from: " + url.getFile) - val properties = new Properties() - var inputStream: InputStream = null - - var reader: InputStreamReader = null - var buff: BufferedReader = null - - Utils.tryFinally { - Utils.tryThrow { - inputStream = new FileInputStream(new File(url.getFile)) - reader = new InputStreamReader(inputStream, "UTF-8") - buff = new BufferedReader(reader) - properties.load(buff) - } { - case t: Throwable => - new AnomalyScannerException( - 21304, - "Failed to load alerts from alert properties. Cause: " + ExceptionUtils.getMessage(t) - ) - } - } { - IOUtils.closeQuietly(buff) - IOUtils.closeQuietly(reader) - IOUtils.closeQuietly(inputStream) - } - for ((k: String, v: String) <- properties.asScala) { - if (ret.containsKey(k)) { - logger.warn("found duplicate key in alert properties, accept only the first one") - } else if (StringUtils.startsWith(k, prefix)) { - val data = mapper.readValue(v, classOf[ImsAlertPropFileData]) - val alertInfo = - new String(new StringBuilder().append(data.alertInfo).toString().getBytes(), "utf-8") - .replace("$userCreator", instans) - val receivers = { - val set: util.Set[String] = new util.HashSet[String] - if (StringUtils.isNotBlank(data.alertReceivers)) { - data.alertReceivers.split(",").map(r => set.add(r)) - } - Constants.ALERT_DEFAULT_RECEIVERS.foreach(e => { - if (StringUtils.isNotBlank(e)) { - set.add(e) - } - }) - set - } - val alertDesc = Utils.tryAndWarn( - new ImsAlertDesc( - Constants.ALERT_SUB_SYSTEM_ID, - data.alertTitle, - data.alertObj, - alertInfo, - ImsAlertLevel.withName(data.alertLevel), - null, // Not used - 0, { - val set: util.Set[ImsAlertWay.Value] = new util.HashSet[ImsAlertWay.Value] - if (StringUtils.isNotBlank(data.alertWays)) { - data.alertWays - .split(",") - .map(alertWayStr => set.add(ImsAlertWay.withName(alertWayStr))) - } - set - }, - receivers - ) - ) - val realK = StringUtils.substringAfter(k, prefix) - ret.put(realK, alertDesc) - } - } - ret + def getAlerts(prefix: String, userCreator: String): util.Map[String, AlertDesc] = { + val replaceParams: util.HashMap[String, String] = new util.HashMap[String, String] + replaceParams.put("$userCreator", userCreator) + MonitorAlertUtils.getAlerts(prefix, replaceParams) } } diff --git a/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/utils/log/LogUtils.scala b/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/utils/log/LogUtils.scala index 38af7b6104..73daf86721 100644 --- a/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/utils/log/LogUtils.scala +++ b/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/utils/log/LogUtils.scala @@ -19,7 +19,6 @@ package org.apache.linkis.monitor.scan.utils.log import org.slf4j.LoggerFactory - object LogUtils { val stdOutLogger = LoggerFactory.getLogger("PlaintTextConsoleLogger") } diff --git a/linkis-extensions/linkis-et-monitor/src/test/java/org/apache/linkis/monitor/scan/app/JobHistoryScanImsAlertPropFileParserUtilsTest.java b/linkis-extensions/linkis-et-monitor/src/test/java/org/apache/linkis/monitor/scan/app/JobHistoryScanImsAlertPropFileParserUtilsTest.java deleted file mode 100644 index 600ccafddb..0000000000 --- a/linkis-extensions/linkis-et-monitor/src/test/java/org/apache/linkis/monitor/scan/app/JobHistoryScanImsAlertPropFileParserUtilsTest.java +++ /dev/null @@ -1,48 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.monitor.scan.app; - -import org.apache.linkis.monitor.scan.constants.Constants; -import org.apache.linkis.monitor.scan.utils.alert.AlertDesc; -import org.apache.linkis.monitor.scan.utils.alert.ims.JobHistoryScanImsAlertPropFileParserUtils; -import org.apache.linkis.server.utils.LinkisMainHelper; - -import java.util.Map; - -import org.junit.Assert; - -public class JobHistoryScanImsAlertPropFileParserUtilsTest { - // @Before - public void before() { - System.getProperties().setProperty(LinkisMainHelper.SERVER_NAME_KEY(), "linkis-et-monitor"); - System.getProperties() - .setProperty("log4j.configurationFile", "src/test/resources/log4j2-console.xml"); - // System.getProperties().setProperty("wds.linkis.server.conf", - // "linkis-et-monitor.properties"); - } - - // @Test - public void getAlerts() throws Exception { - Map alerts = - JobHistoryScanImsAlertPropFileParserUtils.getAlerts(Constants.SCAN_PREFIX_ERRORCODE()); - for (Map.Entry kv : alerts.entrySet()) { - System.out.println(kv.getKey() + ": " + kv.getValue().toString()); - } - Assert.assertEquals(alerts.size(), 2); - } -} diff --git a/linkis-extensions/linkis-et-monitor/src/test/java/org/apache/linkis/monitor/scan/utils/alert/PooledImsAlertUtilsTest.java b/linkis-extensions/linkis-et-monitor/src/test/java/org/apache/linkis/monitor/scan/utils/alert/PooledImsAlertUtilsTest.java index 8ca5ff1a98..2014058561 100644 --- a/linkis-extensions/linkis-et-monitor/src/test/java/org/apache/linkis/monitor/scan/utils/alert/PooledImsAlertUtilsTest.java +++ b/linkis-extensions/linkis-et-monitor/src/test/java/org/apache/linkis/monitor/scan/utils/alert/PooledImsAlertUtilsTest.java @@ -18,7 +18,7 @@ package org.apache.linkis.monitor.scan.utils.alert; import org.apache.linkis.monitor.scan.constants.Constants; -import org.apache.linkis.monitor.scan.utils.alert.ims.JobHistoryScanImsAlertPropFileParserUtils; +import org.apache.linkis.monitor.scan.utils.alert.ims.MonitorAlertUtils; import org.apache.linkis.monitor.scan.utils.alert.ims.PooledImsAlertUtils; import org.apache.linkis.server.utils.LinkisMainHelper; @@ -38,7 +38,7 @@ public void before() { public void addAlert() throws Exception { PooledImsAlertUtils.addAlert("1st test"); Map alerts = - JobHistoryScanImsAlertPropFileParserUtils.getAlerts(Constants.SCAN_PREFIX_ERRORCODE()); + MonitorAlertUtils.getAlerts((Constants.SCAN_PREFIX_ERRORCODE()), null); for (Map.Entry kv : alerts.entrySet()) { System.out.println(kv.getKey() + ": " + kv.getValue().toString()); PooledImsAlertUtils.addAlert(kv.getValue()); From 5e52f909e4c06aedb4c5e9ab7fa2c8bae7f23884 Mon Sep 17 00:00:00 2001 From: peacewong Date: Fri, 8 Sep 2023 18:57:48 +0800 Subject: [PATCH 3/6] add bml --- linkis-extensions/linkis-et-monitor/pom.xml | 24 +- .../scan/app/bml/cleaner/dao/VersionDao.java | 61 +++++ .../entity/CleanedResourceVersion.java | 209 ++++++++++++++++++ .../bml/cleaner/entity/ResourceVersion.java | 206 +++++++++++++++++ .../bml/cleaner/service/CleanerService.java | 23 ++ .../bml/cleaner/service/VersionService.java | 37 ++++ .../service/impl/CleanerServiceImpl.java | 170 ++++++++++++++ .../service/impl/VersionServiceImpl.java | 63 ++++++ .../app/bml/cleaner/vo/CleanResourceVo.java | 48 ++++ .../scan/app/monitor/scheduled/BmlClear.java | 2 +- .../monitor/scan/constants/Constants.scala | 19 ++ .../utils/alert/PooledImsAlertSenderTest.java | 2 +- .../linkis-et-jobhistory-scan-ims.properties | 6 - .../linkis-et-jobhistory-scan.properties | 14 -- 14 files changed, 842 insertions(+), 42 deletions(-) create mode 100644 linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/bml/cleaner/dao/VersionDao.java create mode 100644 linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/bml/cleaner/entity/CleanedResourceVersion.java create mode 100644 linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/bml/cleaner/entity/ResourceVersion.java create mode 100644 linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/bml/cleaner/service/CleanerService.java create mode 100644 linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/bml/cleaner/service/VersionService.java create mode 100644 linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/bml/cleaner/service/impl/CleanerServiceImpl.java create mode 100644 linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/bml/cleaner/service/impl/VersionServiceImpl.java create mode 100644 linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/bml/cleaner/vo/CleanResourceVo.java delete mode 100644 linkis-extensions/linkis-et-monitor/src/test/resources/linkis-et-jobhistory-scan-ims.properties delete mode 100644 linkis-extensions/linkis-et-monitor/src/test/resources/linkis-et-jobhistory-scan.properties diff --git a/linkis-extensions/linkis-et-monitor/pom.xml b/linkis-extensions/linkis-et-monitor/pom.xml index 6972c9e1a4..0e5f44c076 100644 --- a/linkis-extensions/linkis-et-monitor/pom.xml +++ b/linkis-extensions/linkis-et-monitor/pom.xml @@ -36,13 +36,13 @@ org.apache.linkis - linkis-wedatasphere-common + linkis-storage ${project.version} + provided - org.apache.linkis - linkis-module + linkis-rpc ${project.version} provided @@ -68,7 +68,6 @@ junit junit - 4.12 test @@ -76,22 +75,7 @@ linkis-computation-client ${project.version} - - org.apache.linkis - linkis-bml-cleaner - ${project.version} - - - org.apache.linkis - linkis-storage - ${project.version} - provided - - - org.apache.linkis - linkis-rpc - ${project.version} - + diff --git a/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/bml/cleaner/dao/VersionDao.java b/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/bml/cleaner/dao/VersionDao.java new file mode 100644 index 0000000000..6b6a101b45 --- /dev/null +++ b/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/bml/cleaner/dao/VersionDao.java @@ -0,0 +1,61 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.monitor.scan.app.bml.cleaner.dao; + +import org.apache.linkis.monitor.scan.app.bml.cleaner.entity.CleanedResourceVersion; +import org.apache.linkis.monitor.scan.app.bml.cleaner.entity.ResourceVersion; +import org.apache.linkis.monitor.scan.app.bml.cleaner.vo.CleanResourceVo; + +import org.apache.ibatis.annotations.*; + +import java.util.Date; +import java.util.List; + +public interface VersionDao { + + @Select( + "select resource_id, count(resource_id) as version_count, max(version) as max_version from " + + "linkis_ps_bml_resources_version lpbrv where start_time < #{startTime} GROUP BY resource_id HAVING count(resource_id) > #{maxVersionNum} limit #{limitNum}") + List getAllNeedCleanResource( + @Param("maxVersionNum") Integer maxVersionNum, + @Param("startTime") Date startTime, + @Param("limitNum") int num); + + @Select( + "select * from linkis_ps_bml_resources_version where resource_id = #{resourceId} and version < #{minKeepVersion} and version <> 'v000001'") + List getCleanVersionsByResourceId( + @Param("resourceId") String resourceId, @Param("minKeepVersion") String minKeepVersion); + + @Insert({ + "insert into linkis_ps_bml_cleaned_resources_version(`resource_id`,`file_md5`,`version`,`size`,`start_byte`, `end_byte`,`resource`,`description`," + + "`start_time`,`end_time`,`client_ip`,`updator`,`enable_flag`,`old_resource`) values(#{resourceId},#{fileMd5},#{version},#{size},#{startByte},#{endByte}" + + ",#{resource},#{description},#{startTime},#{endTime},#{clientIp},#{updator},#{enableFlag},#{oldResource})" + }) + @Options(useGeneratedKeys = true, keyProperty = "id") + void insertCleanResourceVersion(CleanedResourceVersion cleanedResourceVersion); + + @Delete("delete from linkis_ps_bml_resources_version where id=#{id}") + void deleteResourceVersionById(@Param("id") long id); + + @Select( + "select version from linkis_ps_bml_resources_version where resource_id =#{resourceId} and version <= #{maxVersion} order by version desc limit #{keepNum},1") + String getMinKeepVersion( + @Param("resourceId") String resourceId, + @Param("maxVersion") String maxVersion, + @Param("keepNum") int keepNum); +} diff --git a/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/bml/cleaner/entity/CleanedResourceVersion.java b/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/bml/cleaner/entity/CleanedResourceVersion.java new file mode 100644 index 0000000000..3c6f103183 --- /dev/null +++ b/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/bml/cleaner/entity/CleanedResourceVersion.java @@ -0,0 +1,209 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.monitor.scan.app.bml.cleaner.entity; + +import java.util.Date; + +public class CleanedResourceVersion { + + private long id; + + private String resourceId; + + private String fileMd5; + + private String version; + + private long size; + + private String resource; + + private String oldResource; + + private String description; + + private String clientIp; + + private boolean enableFlag; + + private String user; + + private String system; + + private Date startTime; + + private Date endTime; + + private long startByte; + + private long endByte; + + private String updator; + + public String getResourceId() { + return resourceId; + } + + public void setResourceId(String resourceId) { + this.resourceId = resourceId; + } + + public String getUser() { + return user; + } + + public void setUser(String user) { + this.user = user; + } + + public String getSystem() { + return system; + } + + public void setSystem(String system) { + this.system = system; + } + + public String getVersion() { + return version; + } + + public void setVersion(String version) { + this.version = version; + } + + public String getResource() { + return resource; + } + + public void setResource(String resource) { + this.resource = resource; + } + + public String getOldResource() { + return oldResource; + } + + public void setOldResource(String oldResource) { + this.oldResource = oldResource; + } + + public long getId() { + return id; + } + + public void setId(long id) { + this.id = id; + } + + public String getFileMd5() { + return fileMd5; + } + + public void setFileMd5(String fileMd5) { + this.fileMd5 = fileMd5; + } + + public long getSize() { + return size; + } + + public void setSize(long size) { + this.size = size; + } + + public String getDescription() { + return description; + } + + public void setDescription(String description) { + this.description = description; + } + + public String getClientIp() { + return clientIp; + } + + public void setClientIp(String clientIp) { + this.clientIp = clientIp; + } + + public boolean isEnableFlag() { + return enableFlag; + } + + public void setEnableFlag(boolean enableFlag) { + this.enableFlag = enableFlag; + } + + public long getStartByte() { + return startByte; + } + + public void setStartByte(long startByte) { + this.startByte = startByte; + } + + public long getEndByte() { + return endByte; + } + + public void setEndByte(long endByte) { + this.endByte = endByte; + } + + public Date getStartTime() { + return startTime; + } + + public void setStartTime(Date startTime) { + this.startTime = startTime; + } + + public Date getEndTime() { + return endTime; + } + + public void setEndTime(Date endTime) { + this.endTime = endTime; + } + + public String getUpdator() { + return updator; + } + + public void setUpdator(String updator) { + this.updator = updator; + } + + public static CleanedResourceVersion copyFromResourceVersion(ResourceVersion resourceVersion) { + CleanedResourceVersion cleanedResourceVersion = new CleanedResourceVersion(); + cleanedResourceVersion.setResourceId(resourceVersion.getResourceId()); + cleanedResourceVersion.setOldResource(resourceVersion.getResource()); + cleanedResourceVersion.setFileMd5(resourceVersion.getFileMd5()); + cleanedResourceVersion.setClientIp(resourceVersion.getClientIp()); + cleanedResourceVersion.setSize(resourceVersion.getSize()); + cleanedResourceVersion.setEnableFlag(resourceVersion.getEnableFlag()); + cleanedResourceVersion.setVersion(resourceVersion.getVersion()); + cleanedResourceVersion.setStartByte(resourceVersion.getStartByte()); + cleanedResourceVersion.setEndByte(resourceVersion.getEndByte()); + cleanedResourceVersion.setStartTime(resourceVersion.getStartTime()); + cleanedResourceVersion.setEndTime(resourceVersion.getEndTime()); + return cleanedResourceVersion; + } +} diff --git a/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/bml/cleaner/entity/ResourceVersion.java b/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/bml/cleaner/entity/ResourceVersion.java new file mode 100644 index 0000000000..604ab3662a --- /dev/null +++ b/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/bml/cleaner/entity/ResourceVersion.java @@ -0,0 +1,206 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.monitor.scan.app.bml.cleaner.entity; + +import java.util.Date; + +public class ResourceVersion { + + private long id; + + private String resourceId; + + private String fileMd5; + + private String version; + + private long size; + + private String resource; + + private String description; + + private String clientIp; + + private boolean enableFlag; + + private String user; + + private String system; + + private Date startTime; + + private Date endTime; + + private long startByte; + + private long endByte; + + private String updator; + + public String getResourceId() { + return resourceId; + } + + public void setResourceId(String resourceId) { + this.resourceId = resourceId; + } + + public String getUser() { + return user; + } + + public void setUser(String user) { + this.user = user; + } + + public String getSystem() { + return system; + } + + public void setSystem(String system) { + this.system = system; + } + + public String getVersion() { + return version; + } + + public void setVersion(String version) { + this.version = version; + } + + public String getResource() { + return resource; + } + + public void setResource(String resource) { + this.resource = resource; + } + + public long getId() { + return id; + } + + public void setId(long id) { + this.id = id; + } + + public String getFileMd5() { + return fileMd5; + } + + public void setFileMd5(String fileMd5) { + this.fileMd5 = fileMd5; + } + + public long getSize() { + return size; + } + + public void setSize(long size) { + this.size = size; + } + + public String getDescription() { + return description; + } + + public void setDescription(String description) { + this.description = description; + } + + public String getClientIp() { + return clientIp; + } + + public void setClientIp(String clientIp) { + this.clientIp = clientIp; + } + + public boolean getEnableFlag() { + return enableFlag; + } + + public void setEnableFlag(boolean enableFlag) { + this.enableFlag = enableFlag; + } + + public long getStartByte() { + return startByte; + } + + public void setStartByte(long startByte) { + this.startByte = startByte; + } + + public long getEndByte() { + return endByte; + } + + public void setEndByte(long endByte) { + this.endByte = endByte; + } + + public Date getStartTime() { + return startTime; + } + + public void setStartTime(Date startTime) { + this.startTime = startTime; + } + + public Date getEndTime() { + return endTime; + } + + public void setEndTime(Date endTime) { + this.endTime = endTime; + } + + public String getUpdator() { + return updator; + } + + public void setUpdator(String updator) { + this.updator = updator; + } + + public static ResourceVersion createNewResourceVersion( + String resourceId, + String resourcePath, + String fileMd5, + String clientIp, + long size, + String version, + long startByte) { + ResourceVersion resourceVersion = new ResourceVersion(); + resourceVersion.setResourceId(resourceId); + resourceVersion.setResource(resourcePath); + resourceVersion.setFileMd5(fileMd5); + resourceVersion.setClientIp(clientIp); + resourceVersion.setSize(size); + resourceVersion.setEnableFlag(true); + resourceVersion.setVersion(version); + resourceVersion.setStartByte(startByte); + resourceVersion.setEndByte(startByte + size - 1); + resourceVersion.setStartTime(new Date(System.currentTimeMillis())); + resourceVersion.setEndTime(new Date(System.currentTimeMillis())); + return resourceVersion; + } +} diff --git a/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/bml/cleaner/service/CleanerService.java b/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/bml/cleaner/service/CleanerService.java new file mode 100644 index 0000000000..ccbea7d8c7 --- /dev/null +++ b/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/bml/cleaner/service/CleanerService.java @@ -0,0 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.monitor.scan.app.bml.cleaner.service; + +public interface CleanerService { + + public void run(); +} diff --git a/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/bml/cleaner/service/VersionService.java b/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/bml/cleaner/service/VersionService.java new file mode 100644 index 0000000000..3162c096e9 --- /dev/null +++ b/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/bml/cleaner/service/VersionService.java @@ -0,0 +1,37 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.monitor.scan.app.bml.cleaner.service; + +import org.apache.linkis.monitor.scan.app.bml.cleaner.entity.CleanedResourceVersion; +import org.apache.linkis.common.io.FsPath; +import org.apache.linkis.storage.fs.FileSystem; + +import java.io.IOException; + +public interface VersionService { + + void doMove( + FileSystem fs, + FsPath srcPath, + FsPath destPath, + CleanedResourceVersion insertVersion, + long delVersionId) + throws IOException; + + void moveOnDb(CleanedResourceVersion insertVersion, long delVersionId); +} diff --git a/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/bml/cleaner/service/impl/CleanerServiceImpl.java b/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/bml/cleaner/service/impl/CleanerServiceImpl.java new file mode 100644 index 0000000000..e17d55b7dd --- /dev/null +++ b/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/bml/cleaner/service/impl/CleanerServiceImpl.java @@ -0,0 +1,170 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.monitor.scan.app.bml.cleaner.service.impl; + +import org.apache.linkis.monitor.scan.app.bml.cleaner.dao.VersionDao; +import org.apache.linkis.monitor.scan.app.bml.cleaner.entity.CleanedResourceVersion; +import org.apache.linkis.monitor.scan.app.bml.cleaner.entity.ResourceVersion; +import org.apache.linkis.monitor.scan.app.bml.cleaner.service.CleanerService; +import org.apache.linkis.monitor.scan.app.bml.cleaner.service.VersionService; +import org.apache.linkis.monitor.scan.app.bml.cleaner.vo.CleanResourceVo; +import org.apache.linkis.common.io.FsPath; +import org.apache.linkis.monitor.scan.constants.Constants; +import org.apache.linkis.storage.FSFactory; +import org.apache.linkis.storage.fs.FileSystem; +import org.apache.linkis.storage.utils.StorageConfiguration; +import org.apache.linkis.storage.utils.StorageUtils; + +import org.apache.commons.io.IOUtils; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +import java.io.File; +import java.text.SimpleDateFormat; +import java.util.Date; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import java.util.stream.Collectors; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +@Service +public class CleanerServiceImpl implements CleanerService { + + private final Logger logger = LoggerFactory.getLogger("CleanerServiceImpl"); + + private final SimpleDateFormat sdf = new SimpleDateFormat("yyyyMMdd"); + + public static final String VERSION_FORMAT = "%06d"; + public static final String VERSION_PREFIX = "v"; + public static final String TRASH_DIR = "/trash"; + + private FileSystem fs = null; + + @Autowired private VersionDao versionDao; + + public void setVersionDao(VersionDao versionDao) { + this.versionDao = versionDao; + } + + private Set cleanedResourceIds = new HashSet(); + + Date previous; + + @Autowired VersionService versionService; + + public void clean() { + previous = + new Date( + System.currentTimeMillis() + - (Long) Constants.BML_PREVIOUS_INTERVAL_TIME_DAYS().getValue() + * 86400 + * 1000); + + if ((Integer) Constants.BML_VERSION_MAX_NUM().getValue() + - (Integer) Constants.BML_VERSION_KEEP_NUM().getValue() + <= 1) { + logger.error("conf error need to keep version num > 1"); + return; + } + List needCleanResources = getCleanResources(); + while (needCleanResources != null && needCleanResources.size() > 0) { + logger.info("need cleaned resource count:{}", needCleanResources.size()); + fs = + (FileSystem) + FSFactory.getFs( + StorageUtils.HDFS, StorageConfiguration.HDFS_ROOT_USER.getValue()); + for (CleanResourceVo resourceVo : needCleanResources) { + String minVersion = + versionDao.getMinKeepVersion( + resourceVo.getResourceId(), + resourceVo.getMaxVersion(), + (Integer) Constants.BML_VERSION_KEEP_NUM().getValue() - 1); + List cleanVersionList = + versionDao.getCleanVersionsByResourceId(resourceVo.getResourceId(), minVersion); + // move on hdfs + for (ResourceVersion version : cleanVersionList) { + FsPath srcPath = new FsPath(version.getResource()); + // fs放到外层 + try { + fs.init(null); + if (!fs.exists(srcPath)) { + logger.error("try to move but bml source file:{} not exists!", version.getResource()); + CleanedResourceVersion cleanedResourceVersion = + CleanedResourceVersion.copyFromResourceVersion(version); + cleanedResourceVersion.setResource(""); + versionService.moveOnDb(cleanedResourceVersion, version.getId()); + continue; + } + String destPrefix = + version.getResource().substring(0, version.getResource().indexOf("/bml/") + 4); + String destPath = + destPrefix + + TRASH_DIR + + File.separator + + sdf.format(new Date()) + + File.separator + + version.getResourceId() + + "_" + + version.getVersion(); + FsPath dest = new FsPath(destPath); + if (!fs.exists(dest.getParent())) { + fs.mkdirs(dest.getParent()); + } + logger.info("begin to mv bml resource:{} to dest:{}", version.getResource(), destPath); + CleanedResourceVersion cleanedResourceVersion = + CleanedResourceVersion.copyFromResourceVersion(version); + cleanedResourceVersion.setResource(destPath); + versionService.doMove(fs, srcPath, dest, cleanedResourceVersion, version.getId()); + } catch (Exception e) { + logger.error("failed to mv bml resource:{}", e.getMessage(), e); + } + } + + cleanedResourceIds.add(resourceVo.getResourceId()); + } + needCleanResources = getCleanResources(); + } + } + + public void run() { + logger.info("start to clean."); + clean(); + logger.info("start to shutdown."); + shutdown(); + } + + void shutdown() { + IOUtils.closeQuietly(fs); + } + + List getCleanResources() { + List cleanResourceVoList = + versionDao.getAllNeedCleanResource( + (Integer) Constants.BML_VERSION_MAX_NUM().getValue(), + previous, + (Integer) Constants.BML_CLEAN_ONCE_RESOURCE_LIMIT_NUM().getValue()); + + return cleanResourceVoList.stream() + .filter(cleanResourceVo -> !cleanedResourceIds.contains(cleanResourceVo.getResourceId())) + .collect(Collectors.toList()); + } +} diff --git a/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/bml/cleaner/service/impl/VersionServiceImpl.java b/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/bml/cleaner/service/impl/VersionServiceImpl.java new file mode 100644 index 0000000000..8234d524af --- /dev/null +++ b/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/bml/cleaner/service/impl/VersionServiceImpl.java @@ -0,0 +1,63 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.monitor.scan.app.bml.cleaner.service.impl; + +import org.apache.linkis.monitor.scan.app.bml.cleaner.dao.VersionDao; +import org.apache.linkis.monitor.scan.app.bml.cleaner.entity.CleanedResourceVersion; +import org.apache.linkis.monitor.scan.app.bml.cleaner.service.VersionService; +import org.apache.linkis.common.io.FsPath; +import org.apache.linkis.storage.fs.FileSystem; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; + +import java.io.IOException; + +@Service +public class VersionServiceImpl implements VersionService { + + @Autowired VersionDao versionDao; + + public void setVersionDao(VersionDao versionDao) { + this.versionDao = versionDao; + } + + @Transactional(rollbackFor = Throwable.class) + public void doMove( + FileSystem fs, + FsPath srcPath, + FsPath destPath, + CleanedResourceVersion insertVersion, + long delVersionId) + throws IOException { + versionDao.insertCleanResourceVersion(insertVersion); + versionDao.deleteResourceVersionById(delVersionId); + fs.renameTo(srcPath, destPath); + } + + @Transactional + public void moveOnDb(CleanedResourceVersion insertVersion, long delVersionId) { + versionDao.insertCleanResourceVersion(insertVersion); + versionDao.deleteResourceVersionById(delVersionId); + } + + public String test() { + return "this a test string"; + } +} diff --git a/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/bml/cleaner/vo/CleanResourceVo.java b/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/bml/cleaner/vo/CleanResourceVo.java new file mode 100644 index 0000000000..cfcf6e5ea6 --- /dev/null +++ b/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/bml/cleaner/vo/CleanResourceVo.java @@ -0,0 +1,48 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.monitor.scan.app.bml.cleaner.vo; + +public class CleanResourceVo { + private String resourceId; + private int versionCount; + private String maxVersion; + + public String getResourceId() { + return resourceId; + } + + public void setResourceId(String resourceId) { + this.resourceId = resourceId; + } + + public int getVersionCount() { + return versionCount; + } + + public void setVersionCount(int versionCount) { + this.versionCount = versionCount; + } + + public String getMaxVersion() { + return maxVersion; + } + + public void setMaxVersion(String maxVersion) { + this.maxVersion = maxVersion; + } +} diff --git a/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/monitor/scheduled/BmlClear.java b/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/monitor/scheduled/BmlClear.java index 11ee06226f..362af45abe 100644 --- a/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/monitor/scheduled/BmlClear.java +++ b/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/monitor/scheduled/BmlClear.java @@ -17,7 +17,7 @@ package org.apache.linkis.monitor.scan.app.monitor.scheduled; -import org.apache.linkis.bml.cleaner.service.CleanerService; +import org.apache.linkis.monitor.scan.app.bml.cleaner.service.CleanerService; import org.apache.linkis.monitor.scan.utils.log.LogUtils; import org.springframework.beans.factory.annotation.Autowired; diff --git a/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/constants/Constants.scala b/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/constants/Constants.scala index 89ab330b4d..11ea350af3 100644 --- a/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/constants/Constants.scala +++ b/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/scan/constants/Constants.scala @@ -88,4 +88,23 @@ object Constants { val THREAD_TIME_OUT_IM = "thread.monitor.timeout.im." val JOB_RESULT_IM = "jobhistory.result.monitor.im." + + val BML_VERSION_MAX_NUM: CommonVars[Int] = + CommonVars[Int]("linkis.bml.cleaner.version.max.num", 50) + + val BML_VERSION_KEEP_NUM: CommonVars[Int] = + CommonVars[Int]("linkis.bml.cleaner.version.keep.num", 20) + + val BML_PREVIOUS_INTERVAL_TIME_DAYS: CommonVars[Long] = + CommonVars[Long]("linkis.bml.cleaner.previous.interval.days", 30) + + val BML_CLEAN_ONCE_RESOURCE_LIMIT_NUM: CommonVars[Int] = + CommonVars[Int]("linkis.bml.cleaner.once.limit.num", 100) + + val BML_TRASH_PATH_PREFIX: CommonVars[String] = CommonVars[String]( + "linkis.bml.trash.prefix.path", + "hdfs:///tmp/linkis/trash/bml_trash" + ) + + } diff --git a/linkis-extensions/linkis-et-monitor/src/test/java/org/apache/linkis/monitor/scan/utils/alert/PooledImsAlertSenderTest.java b/linkis-extensions/linkis-et-monitor/src/test/java/org/apache/linkis/monitor/scan/utils/alert/PooledImsAlertSenderTest.java index 8cfad4c989..bd6b0a927e 100644 --- a/linkis-extensions/linkis-et-monitor/src/test/java/org/apache/linkis/monitor/scan/utils/alert/PooledImsAlertSenderTest.java +++ b/linkis-extensions/linkis-et-monitor/src/test/java/org/apache/linkis/monitor/scan/utils/alert/PooledImsAlertSenderTest.java @@ -61,7 +61,7 @@ public void doSendAlert() throws Exception { System.out.println(desc); String url = "http://172.21.0.130:10812/ims_data_access/send_alarm_by_json.do"; - PooledImsAlertSender sender = new PooledImsAlertSender("5136", url, receivers); + PooledImsAlertSender sender = new PooledImsAlertSender(url); sender.doSendAlert(desc); } } diff --git a/linkis-extensions/linkis-et-monitor/src/test/resources/linkis-et-jobhistory-scan-ims.properties b/linkis-extensions/linkis-et-monitor/src/test/resources/linkis-et-jobhistory-scan-ims.properties deleted file mode 100644 index f08d130651..0000000000 --- a/linkis-extensions/linkis-et-monitor/src/test/resources/linkis-et-jobhistory-scan-ims.properties +++ /dev/null @@ -1,6 +0,0 @@ -jobhistory.errorcode.11001={"alert_title":"linkis_alert_test","alert_info":"this is a test for linkis errorcode","alert_way":"3,2","alert_reciver":"shangda","alert_level":"2","alert_obj":"linkis_alert","can_recover":"0"} -jobhistory.errorcode.11002={"alert_title":"linkis_alert_test","alert_info":"this is a test for linkis errorcode","alert_way":"3,2","alert_reciver":"shangda","alert_level":"2","alert_obj":"linkis_alert","can_recover":"0"} -jobhistory.errorcode.12011={"alert_title":"linkis_alert_test","alert_info":"this is a test for linkis errorcode","alert_way":"3,2","alert_reciver":"shangda","alert_level":"2","alert_obj":"linkis_alert","can_recover":"0"} -jobhistory.errorcode.43003={"alert_title":"linkis_alert_test","alert_info":"this is a test for linkis errorcode","alert_way":"3,2","alert_reciver":"shangda","alert_level":"2","alert_obj":"linkis_alert","can_recover":"0"} -jobhistory.errorcode.42011={"alert_title":"linkis_alert_test","alert_info":"this is a test for linkis errorcode","alert_way":"3,2","alert_reciver":"shangda","alert_level":"2","alert_obj":"linkis_alert","can_recover":"0"} -jobhistory.unfinished.time.exceed.sec.300={"alert_title":"linkis_alert_test_job_time","alert_info":"this is a test for linkis 5 job time exceed","alert_way":"3,2","alert_reciver":"shangda","alert_level":"2","alert_obj":"linkis_alert","can_recover":"0"} \ No newline at end of file diff --git a/linkis-extensions/linkis-et-monitor/src/test/resources/linkis-et-jobhistory-scan.properties b/linkis-extensions/linkis-et-monitor/src/test/resources/linkis-et-jobhistory-scan.properties deleted file mode 100644 index 0eca42bd5d..0000000000 --- a/linkis-extensions/linkis-et-monitor/src/test/resources/linkis-et-jobhistory-scan.properties +++ /dev/null @@ -1,14 +0,0 @@ -wds.linkis.server.mybatis.mapperLocations=classpath*:org/apache/linkis/jobhistory/scan/app/jobhistory/dao/impl/*.xml -wds.linkis.server.mybatis.typeAliasesPackage=org.apache.linkis.monitor.scan.app.jobhistory.entity -wds.linkis.server.mybatis.BasePackage=org.apache.linkis.monitor.scan.app.jobhistory.dao -wds.linkis.mysql.is.encrypt=false -#wds.linkis.server.mybatis.datasource.url=jdbc:mysql://10.108.161.105:15304/bdp_easy_ide?characterEncoding=UTF-8 -#wds.linkis.server.mybatis.datasource.username=bdpeasyide -#wds.linkis.server.mybatis.datasource.password=bdpeasyide@bdpsit -wds.linkis.server.mybatis.datasource.url=jdbc:mysql://10.107.108.111:3306/vsbi_gz_bdap_sit_01?characterEncoding=UTF-8 -wds.linkis.server.mybatis.datasource.username=bdp_vsbi -wds.linkis.server.mybatis.datasource.password=bdpVsbi@2019 -wds.linkis.alert.url=http://172.21.0.130:10812/ims_data_access/send_alarm_by_json.do -wds.linkis.alert.receiver.default=johnnwang -wds.linkis.errorcode.scanner.interval.seconds=7200 -wds.linkis.errorcode.scanner.max.interval.seconds=3600 \ No newline at end of file From 703eb501e781a509ad5ec615eaa462598bbe82d1 Mon Sep 17 00:00:00 2001 From: peacewong Date: Mon, 11 Sep 2023 22:30:05 +0800 Subject: [PATCH 4/6] code optimize --- .../conf/linkis-et-monitor-ims.properties | 143 ++++++++++++++++++ .../package/conf/linkis-et-monitor.properties | 86 +++++++++++ .../LinksMonitorResourceApplication.java | 2 +- .../app => }/bml/cleaner/dao/VersionDao.java | 9 +- .../entity/CleanedResourceVersion.java | 2 +- .../bml/cleaner/entity/ResourceVersion.java | 2 +- .../bml/cleaner/service/CleanerService.java | 2 +- .../bml/cleaner/service/VersionService.java | 5 +- .../service/impl/CleanerServiceImpl.java | 18 +-- .../service/impl/VersionServiceImpl.java | 13 +- .../bml/cleaner/vo/CleanResourceVo.java | 2 +- .../monitor => }/config/ListenerConfig.java | 8 +- .../monitor => }/config/MonitorConfig.java | 12 +- .../monitor => }/config/ScheduledConfig.java | 2 +- .../app/monitor => }/entity/EngineEntity.java | 2 +- .../monitor => }/entity/EntranceEntity.java | 2 +- .../app/monitor => }/entity/IndexEntity.java | 4 +- .../instance/dao/InsLabelRelationDao.java | 5 +- .../instance/dao/InstanceInfoDao.java | 5 +- .../instance/dao/InstanceLabelDao.java | 5 +- .../instance/entity/InsPersistenceLabel.java | 2 +- .../entity/InsPersistenceLabelValue.java | 2 +- .../instance/entity/InstanceInfo.java | 2 +- .../{scan/app => }/jobhistory/QueryUtils.java | 2 +- .../jobhistory/dao/JobHistoryMapper.java | 5 +- .../jobhistory/entity/JobHistory.java | 5 +- .../exception/AnomalyScannerException.java | 2 +- .../exception/DirtyDataCleanException.java | 2 +- .../app/monitor/entity/ChatbotEntity.java | 97 ------------ .../app/monitor/scheduled/ValidatorClear.java | 51 ------- .../app/monitor => }/scheduled/BmlClear.java | 10 +- .../monitor => }/scheduled/EcRecordClear.java | 8 +- .../EntranceTaskMonitor.java} | 31 ++-- .../scheduled/JobHistoryClear.java | 10 +- .../scheduled/JobHistoryMonitor.java | 83 +++++----- .../scheduled/ResourceMonitor.java | 22 ++- .../monitor => }/scheduled/TaskLogClear.java | 8 +- .../scheduled/UserModeMonitor.java | 16 +- .../app/monitor => }/until/CacheUtils.java | 2 +- .../app/monitor => }/until/HttpsUntils.java | 33 ++-- .../app/monitor => }/until/ThreadUtils.java | 17 +-- .../mapper/common/InsLabelRelationMapper.xml | 10 +- .../mapper/common/InstanceInfoMapper.xml | 4 +- .../mapper/common/InstanceLabelMapper.xml | 4 +- .../mapper/common/JobHistoryMapper.xml | 6 +- ...kisJobHistoryScanSpringConfiguration.scala | 9 +- .../{scan => }/client/MonitorHTTPClient.scala | 6 +- .../client/MonitorHTTPClientClientImpl.scala | 4 +- .../client/MonitorResourceClient.scala | 4 +- .../client/MonitorResourceClientImpl.scala | 4 +- .../{scan => }/constants/Constants.scala | 39 +++-- .../constants/ScanOperatorEnum.scala | 2 +- .../monitor/{scan => }/core/ob/Event.scala | 2 +- .../monitor/{scan => }/core/ob/Observer.scala | 2 +- .../core/ob/SingleObserverEvent.java | 2 +- .../core/pac/AbstractDataFetcher.scala | 2 +- .../core/pac/AbstractScanRule.scala | 6 +- .../{scan => }/core/pac/BaseScannedData.scala | 2 +- .../{scan => }/core/pac/DataFetcher.scala | 2 +- .../{scan => }/core/pac/ScanBuffer.scala | 2 +- .../{scan => }/core/pac/ScanRule.scala | 6 +- .../{scan => }/core/pac/ScannedData.scala | 2 +- .../core/scanner/AbstractScanner.scala | 34 +++-- .../core/scanner/AnomalyScanner.scala | 8 +- .../core/scanner/DefaultScanner.scala | 4 +- .../app => }/factory/MapperFactory.scala | 11 +- .../jobhistory/JobHistoryDataFetcher.scala | 17 ++- .../errorcode/JobHistoryErrCodeHitEvent.scala | 4 +- .../errorcode/JobHistoryErrCodeRule.scala | 11 +- .../JobHistoryErrorCodeAlertSender.scala | 13 +- .../jobtime/JobTimeExceedAlertSender.scala | 15 +- .../jobtime/JobTimeExceedHitEvent.scala | 4 +- .../jobtime/JobTimeExceedRule.scala | 17 +-- .../labels/JobHistoryLabelsAlertSender.scala | 15 +- .../labels/JobHistoryLabelsHitEvent.scala | 4 +- .../labels/JobHistoryLabelsRule.scala | 15 +- .../runtime/CommonJobRunTimeRule.scala | 14 +- .../runtime/CommonRunTimeAlertSender.scala | 14 +- .../runtime/CommonRunTimeHitEvent.scala | 4 +- .../JobHistoryRunTimeAlertSender.scala | 13 +- .../runtime/JobHistoryRunTimeHitEvent.scala | 4 +- .../runtime/JobHistoryRunTimeRule.scala | 17 +-- .../{scan => }/request/EmsListAction.scala | 2 +- .../request/EntranceTaskAction.scala | 2 +- .../request/MonitorResourceAction.scala | 2 +- .../{scan => }/request/UserAction.scala | 2 +- .../response/EntranceTaskResult.scala | 2 +- .../response/MonitorResourceResult.scala | 2 +- .../monitor/{scan => }/utils/ScanUtils.java | 2 +- .../{scan => }/utils/alert/AlertDesc.scala | 2 +- .../{scan => }/utils/alert/AlertSender.scala | 2 +- .../utils/alert/PooledAlertSender.scala | 2 +- .../utils/alert/ims/ImsAlertDesc.scala | 11 +- .../utils/alert/ims/ImsAlertLevel.scala | 2 +- .../alert/ims/ImsAlertPropFileData.scala | 2 +- .../utils/alert/ims/ImsAlertWay.scala | 2 +- .../utils/alert/ims/ImsRequest.scala | 2 +- .../utils/alert/ims/MonitorAlertUtils.scala | 13 +- .../alert/ims/PooledImsAlertSender.scala | 8 +- .../utils/alert/ims/PooledImsAlertUtils.scala | 6 +- .../utils/alert/ims/UserLabelAlertUtils.scala | 11 +- .../{scan => }/utils/log/LogUtils.scala | 2 +- .../utils/alert/PooledImsAlertSenderTest.java | 10 +- .../utils/alert/PooledImsAlertUtilsTest.java | 8 +- 104 files changed, 620 insertions(+), 574 deletions(-) create mode 100644 linkis-dist/package/conf/linkis-et-monitor-ims.properties create mode 100644 linkis-dist/package/conf/linkis-et-monitor.properties rename linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/{scan/app => }/LinksMonitorResourceApplication.java (96%) rename linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/{scan/app => }/bml/cleaner/dao/VersionDao.java (89%) rename linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/{scan/app => }/bml/cleaner/entity/CleanedResourceVersion.java (98%) rename linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/{scan/app => }/bml/cleaner/entity/ResourceVersion.java (98%) rename linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/{scan/app => }/bml/cleaner/service/CleanerService.java (93%) rename linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/{scan/app => }/bml/cleaner/service/VersionService.java (85%) rename linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/{scan/app => }/bml/cleaner/service/impl/CleanerServiceImpl.java (91%) rename linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/{scan/app => }/bml/cleaner/service/impl/VersionServiceImpl.java (81%) rename linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/{scan/app => }/bml/cleaner/vo/CleanResourceVo.java (95%) rename linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/{scan/app/monitor => }/config/ListenerConfig.java (84%) rename linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/{scan/app/monitor => }/config/MonitorConfig.java (85%) rename linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/{scan/app/monitor => }/config/ScheduledConfig.java (96%) rename linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/{scan/app/monitor => }/entity/EngineEntity.java (95%) rename linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/{scan/app/monitor => }/entity/EntranceEntity.java (96%) rename linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/{scan/app/monitor => }/entity/IndexEntity.java (94%) rename linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/{scan/app => }/instance/dao/InsLabelRelationDao.java (85%) rename linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/{scan/app => }/instance/dao/InstanceInfoDao.java (88%) rename linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/{scan/app => }/instance/dao/InstanceLabelDao.java (85%) rename linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/{scan/app => }/instance/entity/InsPersistenceLabel.java (97%) rename linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/{scan/app => }/instance/entity/InsPersistenceLabelValue.java (96%) rename linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/{scan/app => }/instance/entity/InstanceInfo.java (97%) rename linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/{scan/app => }/jobhistory/QueryUtils.java (95%) rename linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/{scan/app => }/jobhistory/dao/JobHistoryMapper.java (92%) rename linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/{scan/app => }/jobhistory/entity/JobHistory.java (96%) rename linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/{scan/app => }/jobhistory/exception/AnomalyScannerException.java (95%) rename linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/{scan/app => }/jobhistory/exception/DirtyDataCleanException.java (95%) delete mode 100644 linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/monitor/entity/ChatbotEntity.java delete mode 100644 linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/monitor/scheduled/ValidatorClear.java rename linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/{scan/app/monitor => }/scheduled/BmlClear.java (83%) rename linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/{scan/app/monitor => }/scheduled/EcRecordClear.java (87%) rename linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/{scan/app/monitor/scheduled/TaskMonitor.java => scheduled/EntranceTaskMonitor.java} (89%) rename linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/{scan/app/monitor => }/scheduled/JobHistoryClear.java (84%) rename linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/{scan/app/monitor => }/scheduled/JobHistoryMonitor.java (77%) rename linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/{scan/app/monitor => }/scheduled/ResourceMonitor.java (90%) rename linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/{scan/app/monitor => }/scheduled/TaskLogClear.java (87%) rename linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/{scan/app/monitor => }/scheduled/UserModeMonitor.java (94%) rename linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/{scan/app/monitor => }/until/CacheUtils.java (95%) rename linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/{scan/app/monitor => }/until/HttpsUntils.java (82%) rename linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/{scan/app/monitor => }/until/ThreadUtils.java (80%) rename linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/{scan/app => }/LinkisJobHistoryScanSpringConfiguration.scala (85%) rename linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/{scan => }/client/MonitorHTTPClient.scala (95%) rename linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/{scan => }/client/MonitorHTTPClientClientImpl.scala (92%) rename linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/{scan => }/client/MonitorResourceClient.scala (96%) rename linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/{scan => }/client/MonitorResourceClientImpl.scala (92%) rename linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/{scan => }/constants/Constants.scala (70%) rename linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/{scan => }/constants/ScanOperatorEnum.scala (94%) rename linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/{scan => }/core/ob/Event.scala (95%) rename linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/{scan => }/core/ob/Observer.scala (95%) rename linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/{scan => }/core/ob/SingleObserverEvent.java (96%) rename linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/{scan => }/core/pac/AbstractDataFetcher.scala (95%) rename linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/{scan => }/core/pac/AbstractScanRule.scala (89%) rename linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/{scan => }/core/pac/BaseScannedData.scala (95%) rename linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/{scan => }/core/pac/DataFetcher.scala (96%) rename linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/{scan => }/core/pac/ScanBuffer.scala (96%) rename linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/{scan => }/core/pac/ScanRule.scala (88%) rename linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/{scan => }/core/pac/ScannedData.scala (94%) rename linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/{scan => }/core/scanner/AbstractScanner.scala (83%) rename linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/{scan => }/core/scanner/AnomalyScanner.scala (90%) rename linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/{scan => }/core/scanner/DefaultScanner.scala (88%) rename linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/{scan/app => }/factory/MapperFactory.scala (85%) rename linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/{scan/app => }/jobhistory/JobHistoryDataFetcher.scala (86%) rename linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/{scan/app => }/jobhistory/errorcode/JobHistoryErrCodeHitEvent.scala (86%) rename linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/{scan/app => }/jobhistory/errorcode/JobHistoryErrCodeRule.scala (89%) rename linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/{scan/app => }/jobhistory/errorcode/JobHistoryErrorCodeAlertSender.scala (89%) rename linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/{scan/app => }/jobhistory/jobtime/JobTimeExceedAlertSender.scala (87%) rename linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/{scan/app => }/jobhistory/jobtime/JobTimeExceedHitEvent.scala (87%) rename linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/{scan/app => }/jobhistory/jobtime/JobTimeExceedRule.scala (85%) rename linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/{scan/app => }/jobhistory/labels/JobHistoryLabelsAlertSender.scala (83%) rename linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/{scan/app => }/jobhistory/labels/JobHistoryLabelsHitEvent.scala (87%) rename linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/{scan/app => }/jobhistory/labels/JobHistoryLabelsRule.scala (91%) rename linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/{scan/app => }/jobhistory/runtime/CommonJobRunTimeRule.scala (87%) rename linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/{scan/app => }/jobhistory/runtime/CommonRunTimeAlertSender.scala (89%) rename linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/{scan/app => }/jobhistory/runtime/CommonRunTimeHitEvent.scala (87%) rename linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/{scan/app => }/jobhistory/runtime/JobHistoryRunTimeAlertSender.scala (86%) rename linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/{scan/app => }/jobhistory/runtime/JobHistoryRunTimeHitEvent.scala (87%) rename linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/{scan/app => }/jobhistory/runtime/JobHistoryRunTimeRule.scala (85%) rename linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/{scan => }/request/EmsListAction.scala (97%) rename linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/{scan => }/request/EntranceTaskAction.scala (98%) rename linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/{scan => }/request/MonitorResourceAction.scala (95%) rename linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/{scan => }/request/UserAction.scala (95%) rename linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/{scan => }/response/EntranceTaskResult.scala (96%) rename linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/{scan => }/response/MonitorResourceResult.scala (95%) rename linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/{scan => }/utils/ScanUtils.java (97%) rename linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/{scan => }/utils/alert/AlertDesc.scala (94%) rename linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/{scan => }/utils/alert/AlertSender.scala (95%) rename linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/{scan => }/utils/alert/PooledAlertSender.scala (98%) rename linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/{scan => }/utils/alert/ims/ImsAlertDesc.scala (95%) rename linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/{scan => }/utils/alert/ims/ImsAlertLevel.scala (94%) rename linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/{scan => }/utils/alert/ims/ImsAlertPropFileData.scala (95%) rename linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/{scan => }/utils/alert/ims/ImsAlertWay.scala (94%) rename linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/{scan => }/utils/alert/ims/ImsRequest.scala (96%) rename linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/{scan => }/utils/alert/ims/MonitorAlertUtils.scala (94%) rename linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/{scan => }/utils/alert/ims/PooledImsAlertSender.scala (93%) rename linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/{scan => }/utils/alert/ims/PooledImsAlertUtils.scala (95%) rename linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/{scan => }/utils/alert/ims/UserLabelAlertUtils.scala (86%) rename linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/{scan => }/utils/log/LogUtils.scala (94%) rename linkis-extensions/linkis-et-monitor/src/test/java/org/apache/linkis/monitor/{scan => }/utils/alert/PooledImsAlertSenderTest.java (86%) rename linkis-extensions/linkis-et-monitor/src/test/java/org/apache/linkis/monitor/{scan => }/utils/alert/PooledImsAlertUtilsTest.java (87%) diff --git a/linkis-dist/package/conf/linkis-et-monitor-ims.properties b/linkis-dist/package/conf/linkis-et-monitor-ims.properties new file mode 100644 index 0000000000..22a45841d8 --- /dev/null +++ b/linkis-dist/package/conf/linkis-et-monitor-ims.properties @@ -0,0 +1,143 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +jobhistory.errorcode.01002={"alert_title":"Linkis Service load is too high, please contact Linkis owner","alert_info":"Linkis Linkis Service load is too high, please contact Linkis owner","alert_way":"1,2,3","alert_reciver":"hadoop","alert_level":"2","alert_obj":"linkis-alert-obj","can_recover":"0"} +jobhistory.errorcode.01003={"alert_title":"Linkis Service load is too high, please contact Linkis owner","alert_info":"Linkis Service load is too high, please contact Linkis owner","alert_way":"1,2,3","alert_reciver":"hadoop","alert_level":"2","alert_obj":"linkis-alert-obj","can_recover":"0"} +jobhistory.errorcode.01101={"alert_title":"Linkis Service load is too high, please contact Linkis owner","alert_info":" Service load is too high, please contact Linkis owner","alert_way":"1,2,3","alert_reciver":"hadoop","alert_level":"2","alert_obj":"linkis-alert-obj","can_recover":"0"} +jobhistory.errorcode.01102={"alert_title":"Linkis ECM memory Service load is too high, please contact Linkis owner","alert_info":" Service load is too high, please contact Linkis owner","alert_way":"1,2,3","alert_reciver":"hadoop","alert_level":"2","alert_obj":"linkis-alert-obj","can_recover":"0"} +jobhistory.errorcode.01103={"alert_title":"Linkis ECM CPU Service load is too high, please contact Linkis owner","alert_info":" Service load is too high, please contact Linkis owner","alert_way":"1,2,3","alert_reciver":"hadoop","alert_level":"2","alert_obj":"linkis-alert-obj","can_recover":"0"} +jobhistory.errorcode.01104={"alert_title":"Linkis ECM instances Service load is too high, please contact Linkis owner","alert_info":" Service load is too high, please contact Linkis owner","alert_way":"1,2,3","alert_reciver":"hadoop","alert_level":"2","alert_obj":"linkis-alert-obj","can_recover":"0"} +jobhistory.errorcode.01105={"alert_title":"Linkis Memory Service load is too high, please contact Linkis owner","alert_info":"The machine has insufficient memory. Please contact the administrator to expand the memory.","alert_way":"1,2,3","alert_reciver":"hadoop","alert_level":"2","alert_obj":"linkis-alert-obj","can_recover":"0"} +jobhistory.unfinished.time.exceed.sec.43200={"alert_title":"Linkis task execute timeout","alert_info":"Linkis task execute timeout 12h","alert_way":"1,2,3","alert_reciver":"","alert_level":"4","alert_obj":"Linkis","can_recover":"0"} + +ecm.resource.monitor.im.12003=\ + {"alert_title":"Linkis Monitor Resource Alert",\ + "alert_info":" $instance ECM Memory/cpu insufficient resource < 0.2 please contact Linkis owner: $name",\ + "alert_way":"1,2,3",\ + "alert_reciver":"hadoop",\ + "alert_level":"3",\ + "alert_obj":"linkis_alert",\ + "can_recover":"0"} +ecm.resource.monitor.im.12004=\ + {"alert_title":"Linkis Monitor Resource Alert",\ + "alert_info":" $instance ECM Memory/cpu insufficient resource < 0.2 please contact Linkis owner: $name",\ + "alert_way":"1,2,3",\ + "alert_reciver":"hadoop",\ + "alert_level":"2",\ + "alert_obj":"linkis_alert",\ + "can_recover":"0"} +ecm.resource.monitor.im.12005=\ + {"alert_title":"Linkis entrance user running task monitor",\ + "alert_info":"User $username runninng task at linkis ( $url ) > $runningtask ,please attention",\ + "alert_way":"1,2,3",\ + "alert_reciver":"hadoop",\ + "alert_level":"3",\ + "alert_obj":"linkis_alert",\ + "can_recover":"0"} +ecm.resource.monitor.im.12006=\ + {"alert_title":"Linkis entrance user queued task monitor",\ + "alert_info":"User $username queued task at linkis ( $url ) > $queuedtask ,please attention",\ + "alert_way":"1,2,3",\ + "alert_reciver":"hadoop",\ + "alert_level":"3",\ + "alert_obj":"linkis_alert",\ + "can_recover":"0"} +ecm.resource.monitor.im.12007=\ + {"alert_title":"Linkis entrance user total task monitor",\ + "alert_info":"User $username queued task at linkis ( $url ) > $tasktotal ,please attention",\ + "alert_way":"1,2,3",\ + "alert_reciver":"hadoop",\ + "alert_level":"3",\ + "alert_obj":"linkis_alert",\ + "can_recover":"0"} +ecm.resource.monitor.im.12008=\ + {"alert_title":"Linkis entrance all running task monitor",\ + "alert_info":"linkis ( $url ) running task > $taskminor,please attention",\ + "alert_way":"1,2,3",\ + "alert_reciver":"hadoop",\ + "alert_level":"3",\ + "alert_obj":"linkis_alert",\ + "can_recover":"0"} +ecm.resource.monitor.im.12009=\ + {"alert_title":"Linkis entrance all running task monitor",\ + "alert_info":"linkis ( $url ) all task > $taskmajor,please attention",\ + "alert_way":"1,2,3",\ + "alert_reciver":"hadoop",\ + "alert_level":"2",\ + "alert_obj":"linkis_alert",\ + "can_recover":"0"} + + + +user.mode.monitor.im.12011=\ + {"alert_title":"User mode execution timeout alarm",\ + "alert_info":"User mode execution timeout alarm Linkis url: $url engineType:$engineType Task ID: $jobId please attention $name",\ + "alert_way":"1,2,3",\ + "alert_reciver":"alexyang,hadoop",\ + "alert_level":"2",\ + "alert_obj":"linkis_alert",\ + "can_recover":"0"} +user.mode.monitor.im.12012=\ + {"alert_title":"User mode execution failure alarm",\ + "alert_info":"User mode execution failure alarm Linkis url: $url Engine: $engineType TaskID: $jobId ER=rrorCode?$errorCode errorMsg: $errorMsg please attention $name",\ + "alert_way":"1,2,3",\ + "alert_reciver":"alexyang,hadoop",\ + "alert_level":"2",\ + "alert_obj":"linkis_alert",\ + "can_recover":"0"} + +user.mode.monitor.im.12017=\ + {"alert_title":"@alert_title",\ + "alert_info":"task execute failed, reason $msg",\ + "alert_way":"1,2,3",\ + "alert_reciver":"@alert_reciver",\ + "alert_level":"@alert_level",\ + "alert_obj":"linkis_alert",\ + "can_recover":"0"} +user.mode.monitor.im.12018=\ + {"alert_title":"@alert_title",\ + "alert_info":"task execute time out $timeout",\ + "alert_way":"1,2,3",\ + "alert_reciver":"@alert_reciver",\ + "alert_level":"@alert_level",\ + "alert_obj":"linkis_alert",\ + "can_recover":"0"} + +thread.monitor.timeout.im.12014=\ + {"alert_title":" Linkis Shell Timeout Alert ",\ + "alert_info":"Monitor Shell execute time out $shellName",\ + "alert_way":"1,2,3",\ + "alert_reciver":"hadoop",\ + "alert_level":"3",\ + "alert_obj":"linkis_alert",\ + "can_recover":"0"} +jobhistory.result.monitor.im.12015=\ + {"alert_title":"Linkis \u4EFB\u52A1\u72B6\u6001\u901A\u77E5",\ + "alert_info":"Task ID:$id final status: $status",\ + "alert_way":"1,2,3",\ + "alert_reciver":"",\ + "alert_level":"4",\ + "alert_obj":"linkis_alert",\ + "can_recover":"0"} +jobhistory.result.monitor.im.12016=\ + {"alert_title":"@alert_title",\ + "alert_info":"[Alarm time]$date\n[Subsystem]$sysid\n[Alarm IP]$ip\n[Alarm object]$object\n[Alarm information]$detail",\ + "alert_way":"1,2,3",\ + "alert_reciver":"@alert_reciver",\ + "alert_level":"@alert_level",\ + "alert_obj":"linkis_alert",\ + "can_recover":"0"} + + diff --git a/linkis-dist/package/conf/linkis-et-monitor.properties b/linkis-dist/package/conf/linkis-et-monitor.properties new file mode 100644 index 0000000000..92fcc188af --- /dev/null +++ b/linkis-dist/package/conf/linkis-et-monitor.properties @@ -0,0 +1,86 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +wds.linkis.server.mybatis.typeAliasesPackage=org.apache.linkis.monitor.scan.app.instance.entity,org.apache.linkis.monitor.scan.app.jobhistory.entity,org.apache.linkis.bml.cleaner.dao +wds.linkis.server.mybatis.BasePackage=org.apache.linkis.monitor.scan.app.instance.dao,org.apache.linkis.monitor.scan.app.jobhistory.dao,org.apache.linkis.bml.cleaner.dao + + +# alert server url +linkis.alert.url=http://127.0.0.1/ + +# alert receiver +linkis.alert.receiver.default=hadoop + +# monitor ecm resource +linkis.monitor.ecm.resource.cron=0 0/10 * * * ? + +# Resource remaining ratio +linkis.monitor.ecmResourceTask.major=0.005 +linkis.monitor.ecmResourceTask.minor=0.01 + +# entrance task metrics cron +linkis.monitor.entrance.task.cron=0 0/10 * * * ? + +# Timeout task cron +linkis.monitor.jobHistory.timeout.cron=0 0/20 * * * ? + +# time out interval 24h +linkis.monitor.scanner.timeout.interval.seconds=86400 + + +# Finished task cron +linkis.monitor.jobHistory.finished.cron=0 0/20 * * * ? + +# linkis user mode cron +linkis.monitor.user.mode.cron=0 0/20 * * * ? + +# user mode for engine +linkis.monitor.user.enginelist=[{"engineType":"hive-2.3.3","code":"show tables","runType":"hql","executeUser":"hadoop"},\ + {"engineType":"spark-2.4.3","code":"show tables","runType":"sql","executeUser":"hadoop"},\ + {"engineType":"shell-1","code":"pwd","runType":"sh","executeUser":"hadoop"}] + +linkis.monitor.user.authToken=BML-AUTH + +linkis.monitor.user.mode.timeOut=300 + + +# bml clear cron +linkis.monitor.bml.clear.history.version.cron=0 0 12 * * ? +# bml max version +linkis.bml.cleaner.version.max.num=50 +# keep verssion +linkis.bml.cleaner.version.keep.num=20 +# clear max interval +linkis.bml.cleaner.previous.interval.days=30 +# once scan limit +linkis.bml.cleaner.once.limit.num=100 + +# clear db ec record cron +linkis.monitor.clear.ecRecord.cron=0 10 12 * * ? + +# clear task log cron +linkis.monitor.clear.taskLog.cron=0 10 12 * * ? + +# clear db task history cron +linkis.monitor.clear.history.task.cron=0 0 13 * * ? + +# poll size +linkis.monitor.scheduled.pool.cores.num=20 + +# shell time out conf minute +linkis.monitor.shell.time.out.minute=60 + +##Spring +spring.server.port=9119 diff --git a/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/LinksMonitorResourceApplication.java b/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/LinksMonitorResourceApplication.java similarity index 96% rename from linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/LinksMonitorResourceApplication.java rename to linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/LinksMonitorResourceApplication.java index 405d94037b..941c722f77 100644 --- a/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/LinksMonitorResourceApplication.java +++ b/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/LinksMonitorResourceApplication.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.linkis.monitor.scan.app; +package org.apache.linkis.monitor; import org.apache.linkis.LinkisBaseServerApp; diff --git a/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/bml/cleaner/dao/VersionDao.java b/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/bml/cleaner/dao/VersionDao.java similarity index 89% rename from linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/bml/cleaner/dao/VersionDao.java rename to linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/bml/cleaner/dao/VersionDao.java index 6b6a101b45..5f498500eb 100644 --- a/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/bml/cleaner/dao/VersionDao.java +++ b/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/bml/cleaner/dao/VersionDao.java @@ -15,11 +15,12 @@ * limitations under the License. */ -package org.apache.linkis.monitor.scan.app.bml.cleaner.dao; +package org.apache.linkis.monitor.bml.cleaner.dao; -import org.apache.linkis.monitor.scan.app.bml.cleaner.entity.CleanedResourceVersion; -import org.apache.linkis.monitor.scan.app.bml.cleaner.entity.ResourceVersion; -import org.apache.linkis.monitor.scan.app.bml.cleaner.vo.CleanResourceVo; +import org.apache.linkis.monitor.bml.cleaner.vo.CleanResourceVo; +import org.apache.linkis.monitor.bml.cleaner.entity.CleanedResourceVersion; +import org.apache.linkis.monitor.bml.cleaner.entity.ResourceVersion; +import org.apache.linkis.monitor.bml.cleaner.vo.CleanResourceVo; import org.apache.ibatis.annotations.*; diff --git a/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/bml/cleaner/entity/CleanedResourceVersion.java b/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/bml/cleaner/entity/CleanedResourceVersion.java similarity index 98% rename from linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/bml/cleaner/entity/CleanedResourceVersion.java rename to linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/bml/cleaner/entity/CleanedResourceVersion.java index 3c6f103183..5a0bfcc487 100644 --- a/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/bml/cleaner/entity/CleanedResourceVersion.java +++ b/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/bml/cleaner/entity/CleanedResourceVersion.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.linkis.monitor.scan.app.bml.cleaner.entity; +package org.apache.linkis.monitor.bml.cleaner.entity; import java.util.Date; diff --git a/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/bml/cleaner/entity/ResourceVersion.java b/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/bml/cleaner/entity/ResourceVersion.java similarity index 98% rename from linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/bml/cleaner/entity/ResourceVersion.java rename to linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/bml/cleaner/entity/ResourceVersion.java index 604ab3662a..5d297cf180 100644 --- a/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/bml/cleaner/entity/ResourceVersion.java +++ b/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/bml/cleaner/entity/ResourceVersion.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.linkis.monitor.scan.app.bml.cleaner.entity; +package org.apache.linkis.monitor.bml.cleaner.entity; import java.util.Date; diff --git a/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/bml/cleaner/service/CleanerService.java b/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/bml/cleaner/service/CleanerService.java similarity index 93% rename from linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/bml/cleaner/service/CleanerService.java rename to linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/bml/cleaner/service/CleanerService.java index ccbea7d8c7..0dfa15f396 100644 --- a/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/bml/cleaner/service/CleanerService.java +++ b/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/bml/cleaner/service/CleanerService.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.linkis.monitor.scan.app.bml.cleaner.service; +package org.apache.linkis.monitor.bml.cleaner.service; public interface CleanerService { diff --git a/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/bml/cleaner/service/VersionService.java b/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/bml/cleaner/service/VersionService.java similarity index 85% rename from linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/bml/cleaner/service/VersionService.java rename to linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/bml/cleaner/service/VersionService.java index 3162c096e9..5cc2ee217e 100644 --- a/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/bml/cleaner/service/VersionService.java +++ b/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/bml/cleaner/service/VersionService.java @@ -15,9 +15,10 @@ * limitations under the License. */ -package org.apache.linkis.monitor.scan.app.bml.cleaner.service; +package org.apache.linkis.monitor.bml.cleaner.service; -import org.apache.linkis.monitor.scan.app.bml.cleaner.entity.CleanedResourceVersion; +import org.apache.linkis.monitor.bml.cleaner.entity.CleanedResourceVersion; +import org.apache.linkis.monitor.bml.cleaner.entity.CleanedResourceVersion; import org.apache.linkis.common.io.FsPath; import org.apache.linkis.storage.fs.FileSystem; diff --git a/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/bml/cleaner/service/impl/CleanerServiceImpl.java b/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/bml/cleaner/service/impl/CleanerServiceImpl.java similarity index 91% rename from linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/bml/cleaner/service/impl/CleanerServiceImpl.java rename to linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/bml/cleaner/service/impl/CleanerServiceImpl.java index e17d55b7dd..0005cc08fe 100644 --- a/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/bml/cleaner/service/impl/CleanerServiceImpl.java +++ b/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/bml/cleaner/service/impl/CleanerServiceImpl.java @@ -15,16 +15,16 @@ * limitations under the License. */ -package org.apache.linkis.monitor.scan.app.bml.cleaner.service.impl; - -import org.apache.linkis.monitor.scan.app.bml.cleaner.dao.VersionDao; -import org.apache.linkis.monitor.scan.app.bml.cleaner.entity.CleanedResourceVersion; -import org.apache.linkis.monitor.scan.app.bml.cleaner.entity.ResourceVersion; -import org.apache.linkis.monitor.scan.app.bml.cleaner.service.CleanerService; -import org.apache.linkis.monitor.scan.app.bml.cleaner.service.VersionService; -import org.apache.linkis.monitor.scan.app.bml.cleaner.vo.CleanResourceVo; +package org.apache.linkis.monitor.bml.cleaner.service.impl; + +import org.apache.linkis.monitor.bml.cleaner.dao.VersionDao; +import org.apache.linkis.monitor.bml.cleaner.entity.CleanedResourceVersion; +import org.apache.linkis.monitor.bml.cleaner.entity.ResourceVersion; +import org.apache.linkis.monitor.bml.cleaner.service.CleanerService; +import org.apache.linkis.monitor.bml.cleaner.vo.CleanResourceVo; +import org.apache.linkis.monitor.bml.cleaner.service.VersionService; import org.apache.linkis.common.io.FsPath; -import org.apache.linkis.monitor.scan.constants.Constants; +import org.apache.linkis.monitor.constants.Constants; import org.apache.linkis.storage.FSFactory; import org.apache.linkis.storage.fs.FileSystem; import org.apache.linkis.storage.utils.StorageConfiguration; diff --git a/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/bml/cleaner/service/impl/VersionServiceImpl.java b/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/bml/cleaner/service/impl/VersionServiceImpl.java similarity index 81% rename from linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/bml/cleaner/service/impl/VersionServiceImpl.java rename to linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/bml/cleaner/service/impl/VersionServiceImpl.java index 8234d524af..8e78baa31b 100644 --- a/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/bml/cleaner/service/impl/VersionServiceImpl.java +++ b/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/bml/cleaner/service/impl/VersionServiceImpl.java @@ -15,11 +15,13 @@ * limitations under the License. */ -package org.apache.linkis.monitor.scan.app.bml.cleaner.service.impl; +package org.apache.linkis.monitor.bml.cleaner.service.impl; -import org.apache.linkis.monitor.scan.app.bml.cleaner.dao.VersionDao; -import org.apache.linkis.monitor.scan.app.bml.cleaner.entity.CleanedResourceVersion; -import org.apache.linkis.monitor.scan.app.bml.cleaner.service.VersionService; +import org.apache.linkis.monitor.bml.cleaner.dao.VersionDao; +import org.apache.linkis.monitor.bml.cleaner.entity.CleanedResourceVersion; +import org.apache.linkis.monitor.bml.cleaner.dao.VersionDao; +import org.apache.linkis.monitor.bml.cleaner.entity.CleanedResourceVersion; +import org.apache.linkis.monitor.bml.cleaner.service.VersionService; import org.apache.linkis.common.io.FsPath; import org.apache.linkis.storage.fs.FileSystem; @@ -32,7 +34,8 @@ @Service public class VersionServiceImpl implements VersionService { - @Autowired VersionDao versionDao; + @Autowired + VersionDao versionDao; public void setVersionDao(VersionDao versionDao) { this.versionDao = versionDao; diff --git a/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/bml/cleaner/vo/CleanResourceVo.java b/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/bml/cleaner/vo/CleanResourceVo.java similarity index 95% rename from linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/bml/cleaner/vo/CleanResourceVo.java rename to linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/bml/cleaner/vo/CleanResourceVo.java index cfcf6e5ea6..4ef205effd 100644 --- a/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/bml/cleaner/vo/CleanResourceVo.java +++ b/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/bml/cleaner/vo/CleanResourceVo.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.linkis.monitor.scan.app.bml.cleaner.vo; +package org.apache.linkis.monitor.bml.cleaner.vo; public class CleanResourceVo { private String resourceId; diff --git a/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/monitor/config/ListenerConfig.java b/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/config/ListenerConfig.java similarity index 84% rename from linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/monitor/config/ListenerConfig.java rename to linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/config/ListenerConfig.java index fae16b63a3..eb5c11af87 100644 --- a/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/monitor/config/ListenerConfig.java +++ b/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/config/ListenerConfig.java @@ -15,11 +15,11 @@ * limitations under the License. */ -package org.apache.linkis.monitor.scan.app.monitor.config; +package org.apache.linkis.monitor.config; -import org.apache.linkis.monitor.scan.app.monitor.until.HttpsUntils; -import org.apache.linkis.monitor.scan.app.monitor.until.ThreadUtils; -import org.apache.linkis.monitor.scan.utils.log.LogUtils; +import org.apache.linkis.monitor.until.HttpsUntils; +import org.apache.linkis.monitor.until.ThreadUtils; +import org.apache.linkis.monitor.utils.log.LogUtils; import org.springframework.context.annotation.Configuration; import org.springframework.context.event.ContextClosedEvent; diff --git a/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/monitor/config/MonitorConfig.java b/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/config/MonitorConfig.java similarity index 85% rename from linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/monitor/config/MonitorConfig.java rename to linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/config/MonitorConfig.java index 32b47c289c..50516e140c 100644 --- a/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/monitor/config/MonitorConfig.java +++ b/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/config/MonitorConfig.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.linkis.monitor.scan.app.monitor.config; +package org.apache.linkis.monitor.config; import org.apache.linkis.common.conf.CommonVars; import org.apache.linkis.common.conf.Configuration; @@ -29,7 +29,7 @@ public class MonitorConfig { public static final CommonVars USER_MODE_TIMEOUT = CommonVars.apply("linkis.monitor.user.timeOut", 300); public static final CommonVars USER_MODE_AUTHTOKEN = - CommonVars.apply("linkis.monitor.user.authToken", "VALIDATOR-AUTH"); + CommonVars.apply("linkis.monitor.user.authToken", "BML-AUTH"); public static final CommonVars USER_MODE_ENGINE = CommonVars.apply("linkis.monitor.user.enginelist", "[]"); @@ -63,14 +63,6 @@ public class MonitorConfig { public static final CommonVars USER_MODE_INTERFACE_TIMEOUT = CommonVars.apply("linkis.monitor.user.mode.time.out", 30 * 1000); - public static final CommonVars CHATBOT_KEY_ID = - CommonVars.apply("linkis.monitor.chatbot.key.id", "23e6afad1b78a0c5eed67e4d24de7063"); - public static final CommonVars CHATBOT_TYPE = - CommonVars.apply("linkis.monitor.chatbot.type", "text"); - public static final CommonVars CHATBOT_SERVICE_NAME = - CommonVars.apply("linkis.monitor.chatbot.serviceName", ""); - public static final CommonVars CHATBOT_URL = - CommonVars.apply("linkis.monitor.chatbot.url", ""); public static final CommonVars SOLUTION_URL = CommonVars.apply( "linkis.monitor.jobhistory.solution.url", diff --git a/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/monitor/config/ScheduledConfig.java b/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/config/ScheduledConfig.java similarity index 96% rename from linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/monitor/config/ScheduledConfig.java rename to linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/config/ScheduledConfig.java index 30495a87e1..e954122671 100644 --- a/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/monitor/config/ScheduledConfig.java +++ b/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/config/ScheduledConfig.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.linkis.monitor.scan.app.monitor.config; +package org.apache.linkis.monitor.config; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; diff --git a/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/monitor/entity/EngineEntity.java b/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/entity/EngineEntity.java similarity index 95% rename from linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/monitor/entity/EngineEntity.java rename to linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/entity/EngineEntity.java index 650fa081cc..760c06ba1b 100644 --- a/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/monitor/entity/EngineEntity.java +++ b/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/entity/EngineEntity.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.linkis.monitor.scan.app.monitor.entity; +package org.apache.linkis.monitor.entity; import java.io.Serializable; diff --git a/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/monitor/entity/EntranceEntity.java b/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/entity/EntranceEntity.java similarity index 96% rename from linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/monitor/entity/EntranceEntity.java rename to linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/entity/EntranceEntity.java index 6fa9441474..241439b499 100644 --- a/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/monitor/entity/EntranceEntity.java +++ b/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/entity/EntranceEntity.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.linkis.monitor.scan.app.monitor.entity; +package org.apache.linkis.monitor.entity; import java.io.Serializable; diff --git a/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/monitor/entity/IndexEntity.java b/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/entity/IndexEntity.java similarity index 94% rename from linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/monitor/entity/IndexEntity.java rename to linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/entity/IndexEntity.java index fbea8886e8..2905f8aced 100644 --- a/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/monitor/entity/IndexEntity.java +++ b/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/entity/IndexEntity.java @@ -15,9 +15,9 @@ * limitations under the License. */ -package org.apache.linkis.monitor.scan.app.monitor.entity; +package org.apache.linkis.monitor.entity; -import org.apache.linkis.monitor.scan.constants.Constants; +import org.apache.linkis.monitor.constants.Constants; public class IndexEntity { diff --git a/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/instance/dao/InsLabelRelationDao.java b/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/instance/dao/InsLabelRelationDao.java similarity index 85% rename from linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/instance/dao/InsLabelRelationDao.java rename to linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/instance/dao/InsLabelRelationDao.java index 3705288e66..9e573f42d4 100644 --- a/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/instance/dao/InsLabelRelationDao.java +++ b/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/instance/dao/InsLabelRelationDao.java @@ -15,9 +15,10 @@ * limitations under the License. */ -package org.apache.linkis.monitor.scan.app.instance.dao; +package org.apache.linkis.monitor.instance.dao; -import org.apache.linkis.monitor.scan.app.instance.entity.InsPersistenceLabel; +import org.apache.linkis.monitor.instance.entity.InsPersistenceLabel; +import org.apache.linkis.monitor.instance.entity.InsPersistenceLabel; import java.util.List; diff --git a/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/instance/dao/InstanceInfoDao.java b/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/instance/dao/InstanceInfoDao.java similarity index 88% rename from linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/instance/dao/InstanceInfoDao.java rename to linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/instance/dao/InstanceInfoDao.java index 73a7166a5f..136df3a252 100644 --- a/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/instance/dao/InstanceInfoDao.java +++ b/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/instance/dao/InstanceInfoDao.java @@ -15,10 +15,11 @@ * limitations under the License. */ -package org.apache.linkis.monitor.scan.app.instance.dao; +package org.apache.linkis.monitor.instance.dao; import org.apache.linkis.common.ServiceInstance; -import org.apache.linkis.monitor.scan.app.instance.entity.InstanceInfo; +import org.apache.linkis.monitor.instance.entity.InstanceInfo; +import org.apache.linkis.monitor.instance.entity.InstanceInfo; import org.apache.ibatis.annotations.Param; diff --git a/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/instance/dao/InstanceLabelDao.java b/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/instance/dao/InstanceLabelDao.java similarity index 85% rename from linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/instance/dao/InstanceLabelDao.java rename to linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/instance/dao/InstanceLabelDao.java index 5fa3b7a5c0..4476444460 100644 --- a/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/instance/dao/InstanceLabelDao.java +++ b/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/instance/dao/InstanceLabelDao.java @@ -15,9 +15,10 @@ * limitations under the License. */ -package org.apache.linkis.monitor.scan.app.instance.dao; +package org.apache.linkis.monitor.instance.dao; -import org.apache.linkis.monitor.scan.app.instance.entity.InsPersistenceLabel; +import org.apache.linkis.monitor.instance.entity.InsPersistenceLabel; +import org.apache.linkis.monitor.instance.entity.InsPersistenceLabel; import java.util.List; diff --git a/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/instance/entity/InsPersistenceLabel.java b/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/instance/entity/InsPersistenceLabel.java similarity index 97% rename from linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/instance/entity/InsPersistenceLabel.java rename to linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/instance/entity/InsPersistenceLabel.java index 36fb21c13b..0959c27988 100644 --- a/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/instance/entity/InsPersistenceLabel.java +++ b/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/instance/entity/InsPersistenceLabel.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.linkis.monitor.scan.app.instance.entity; +package org.apache.linkis.monitor.instance.entity; import java.util.Date; diff --git a/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/instance/entity/InsPersistenceLabelValue.java b/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/instance/entity/InsPersistenceLabelValue.java similarity index 96% rename from linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/instance/entity/InsPersistenceLabelValue.java rename to linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/instance/entity/InsPersistenceLabelValue.java index 53ab4bd806..029f552608 100644 --- a/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/instance/entity/InsPersistenceLabelValue.java +++ b/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/instance/entity/InsPersistenceLabelValue.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.linkis.monitor.scan.app.instance.entity; +package org.apache.linkis.monitor.instance.entity; public class InsPersistenceLabelValue { diff --git a/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/instance/entity/InstanceInfo.java b/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/instance/entity/InstanceInfo.java similarity index 97% rename from linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/instance/entity/InstanceInfo.java rename to linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/instance/entity/InstanceInfo.java index a74cfaabb0..5360ffd323 100644 --- a/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/instance/entity/InstanceInfo.java +++ b/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/instance/entity/InstanceInfo.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.linkis.monitor.scan.app.instance.entity; +package org.apache.linkis.monitor.instance.entity; import java.util.ArrayList; import java.util.Date; diff --git a/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/jobhistory/QueryUtils.java b/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/jobhistory/QueryUtils.java similarity index 95% rename from linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/jobhistory/QueryUtils.java rename to linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/jobhistory/QueryUtils.java index 799eae89a6..aa73471c49 100644 --- a/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/jobhistory/QueryUtils.java +++ b/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/jobhistory/QueryUtils.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.linkis.monitor.scan.app.jobhistory; +package org.apache.linkis.monitor.jobhistory; import java.text.DateFormat; import java.text.SimpleDateFormat; diff --git a/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/jobhistory/dao/JobHistoryMapper.java b/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/jobhistory/dao/JobHistoryMapper.java similarity index 92% rename from linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/jobhistory/dao/JobHistoryMapper.java rename to linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/jobhistory/dao/JobHistoryMapper.java index bfdc640652..a3cf6fa823 100644 --- a/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/jobhistory/dao/JobHistoryMapper.java +++ b/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/jobhistory/dao/JobHistoryMapper.java @@ -15,9 +15,10 @@ * limitations under the License. */ -package org.apache.linkis.monitor.scan.app.jobhistory.dao; +package org.apache.linkis.monitor.jobhistory.dao; -import org.apache.linkis.monitor.scan.app.jobhistory.entity.JobHistory; +import org.apache.linkis.monitor.jobhistory.entity.JobHistory; +import org.apache.linkis.monitor.jobhistory.entity.JobHistory; import org.apache.ibatis.annotations.Param; diff --git a/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/jobhistory/entity/JobHistory.java b/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/jobhistory/entity/JobHistory.java similarity index 96% rename from linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/jobhistory/entity/JobHistory.java rename to linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/jobhistory/entity/JobHistory.java index 937eff410b..ae378ec202 100644 --- a/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/jobhistory/entity/JobHistory.java +++ b/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/jobhistory/entity/JobHistory.java @@ -15,9 +15,10 @@ * limitations under the License. */ -package org.apache.linkis.monitor.scan.app.jobhistory.entity; +package org.apache.linkis.monitor.jobhistory.entity; -import org.apache.linkis.monitor.scan.app.jobhistory.QueryUtils; +import org.apache.linkis.monitor.jobhistory.QueryUtils; +import org.apache.linkis.monitor.jobhistory.QueryUtils; import java.util.Date; diff --git a/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/jobhistory/exception/AnomalyScannerException.java b/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/jobhistory/exception/AnomalyScannerException.java similarity index 95% rename from linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/jobhistory/exception/AnomalyScannerException.java rename to linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/jobhistory/exception/AnomalyScannerException.java index 0e3dd10d6b..b2c83be013 100644 --- a/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/jobhistory/exception/AnomalyScannerException.java +++ b/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/jobhistory/exception/AnomalyScannerException.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.linkis.monitor.scan.app.jobhistory.exception; +package org.apache.linkis.monitor.jobhistory.exception; import org.apache.linkis.common.exception.ErrorException; import org.apache.linkis.common.exception.ExceptionLevel; diff --git a/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/jobhistory/exception/DirtyDataCleanException.java b/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/jobhistory/exception/DirtyDataCleanException.java similarity index 95% rename from linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/jobhistory/exception/DirtyDataCleanException.java rename to linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/jobhistory/exception/DirtyDataCleanException.java index 93b6065f75..b1c5e64743 100644 --- a/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/jobhistory/exception/DirtyDataCleanException.java +++ b/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/jobhistory/exception/DirtyDataCleanException.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.linkis.monitor.scan.app.jobhistory.exception; +package org.apache.linkis.monitor.jobhistory.exception; import org.apache.linkis.common.exception.ErrorException; import org.apache.linkis.common.exception.ExceptionLevel; diff --git a/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/monitor/entity/ChatbotEntity.java b/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/monitor/entity/ChatbotEntity.java deleted file mode 100644 index 4f912b9e82..0000000000 --- a/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/monitor/entity/ChatbotEntity.java +++ /dev/null @@ -1,97 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.monitor.scan.app.monitor.entity; - -import org.apache.linkis.monitor.scan.app.monitor.config.MonitorConfig; - -public class ChatbotEntity { - private String keyid; - private String content; - private String type; - private String userName; - private String serviceName; - - public ChatbotEntity(String content, String userName) { - this.keyid = MonitorConfig.CHATBOT_KEY_ID.getValue(); - this.content = content; - this.type = MonitorConfig.CHATBOT_TYPE.getValue(); - this.userName = userName; - this.serviceName = MonitorConfig.CHATBOT_SERVICE_NAME.getValue(); - } - - public String getKeyid() { - return keyid; - } - - public void setKeyid(String keyid) { - this.keyid = keyid; - } - - public String getContent() { - return content; - } - - public void setContent(String content) { - this.content = content; - } - - public String getType() { - return type; - } - - public void setType(String type) { - this.type = type; - } - - public String getUserName() { - return userName; - } - - public void setUserName(String userName) { - this.userName = userName; - } - - public String getServiceName() { - return serviceName; - } - - public void setServiceName(String serviceNameuserName) { - this.serviceName = serviceNameuserName; - } - - @Override - public String toString() { - return "ChatbotEntity{" - + "keyid='" - + keyid - + '\'' - + ", content='" - + content - + '\'' - + ", type='" - + type - + '\'' - + ", userName='" - + userName - + '\'' - + ", serviceName='" - + serviceName - + '\'' - + '}'; - } -} diff --git a/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/monitor/scheduled/ValidatorClear.java b/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/monitor/scheduled/ValidatorClear.java deleted file mode 100644 index 4df828e36c..0000000000 --- a/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/monitor/scheduled/ValidatorClear.java +++ /dev/null @@ -1,51 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.monitor.scan.app.monitor.scheduled; - -import org.apache.linkis.monitor.scan.app.monitor.config.MonitorConfig; -import org.apache.linkis.monitor.scan.app.monitor.until.ThreadUtils; -import org.apache.linkis.monitor.scan.utils.log.LogUtils; - -import org.springframework.context.annotation.PropertySource; -import org.springframework.scheduling.annotation.Scheduled; -import org.springframework.stereotype.Component; - -import java.util.ArrayList; -import java.util.List; - -import org.slf4j.Logger; - -/** * Task: clean up linkis_et_validator_checkinfo data */ -@Component -@PropertySource(value = "classpath:linkis-et-monitor.properties", encoding = "UTF-8") -public class ValidatorClear { - - private static final Logger logger = LogUtils.stdOutLogger(); - - @Scheduled(cron = "${linkis.monitor.clear.validator.cron}") - public void ValidatorClear() { - logger.info("Start to clear_validator_record shell"); - List cmdlist = new ArrayList<>(); - cmdlist.add("sh"); - cmdlist.add(MonitorConfig.shellPath + "clear_validator_record.sh"); - logger.info("clear_validator_record shell command {}", cmdlist); - String exec = ThreadUtils.run(cmdlist, "clear_validator_record.sh"); - logger.info("shell log {}", exec); - logger.info("End to clear_validator_record shell "); - } -} diff --git a/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/monitor/scheduled/BmlClear.java b/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scheduled/BmlClear.java similarity index 83% rename from linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/monitor/scheduled/BmlClear.java rename to linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scheduled/BmlClear.java index 362af45abe..cb7d37197f 100644 --- a/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/monitor/scheduled/BmlClear.java +++ b/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scheduled/BmlClear.java @@ -15,18 +15,16 @@ * limitations under the License. */ -package org.apache.linkis.monitor.scan.app.monitor.scheduled; +package org.apache.linkis.monitor.scheduled; -import org.apache.linkis.monitor.scan.app.bml.cleaner.service.CleanerService; -import org.apache.linkis.monitor.scan.utils.log.LogUtils; +import org.apache.linkis.monitor.bml.cleaner.service.CleanerService; +import org.apache.linkis.monitor.utils.log.LogUtils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.annotation.PropertySource; import org.springframework.scheduling.annotation.Scheduled; import org.springframework.stereotype.Component; -import java.util.*; - import org.slf4j.Logger; @Component @@ -37,7 +35,7 @@ public class BmlClear { @Autowired private CleanerService cleanerServices; - @Scheduled(cron = "${linkis.monitor.bml.clear.historyVersion.cron}") + @Scheduled(cron = "${linkis.monitor.bml.clear.history.version.cron}") public void jobHistoryScanTask() { logger.info("start to clear bml history version"); cleanerServices.run(); diff --git a/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/monitor/scheduled/EcRecordClear.java b/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scheduled/EcRecordClear.java similarity index 87% rename from linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/monitor/scheduled/EcRecordClear.java rename to linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scheduled/EcRecordClear.java index ac9cb4ea61..ddb3d3e8c8 100644 --- a/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/monitor/scheduled/EcRecordClear.java +++ b/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scheduled/EcRecordClear.java @@ -15,11 +15,11 @@ * limitations under the License. */ -package org.apache.linkis.monitor.scan.app.monitor.scheduled; +package org.apache.linkis.monitor.scheduled; -import org.apache.linkis.monitor.scan.app.monitor.config.MonitorConfig; -import org.apache.linkis.monitor.scan.app.monitor.until.ThreadUtils; -import org.apache.linkis.monitor.scan.utils.log.LogUtils; +import org.apache.linkis.monitor.config.MonitorConfig; +import org.apache.linkis.monitor.until.ThreadUtils; +import org.apache.linkis.monitor.utils.log.LogUtils; import org.springframework.context.annotation.PropertySource; import org.springframework.scheduling.annotation.Scheduled; diff --git a/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/monitor/scheduled/TaskMonitor.java b/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scheduled/EntranceTaskMonitor.java similarity index 89% rename from linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/monitor/scheduled/TaskMonitor.java rename to linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scheduled/EntranceTaskMonitor.java index 5f6728ef62..d28f6a73f8 100644 --- a/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/monitor/scheduled/TaskMonitor.java +++ b/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scheduled/EntranceTaskMonitor.java @@ -15,16 +15,17 @@ * limitations under the License. */ -package org.apache.linkis.monitor.scan.app.monitor.scheduled; +package org.apache.linkis.monitor.scheduled; import org.apache.linkis.common.ServiceInstance; -import org.apache.linkis.monitor.scan.app.monitor.config.MonitorConfig; -import org.apache.linkis.monitor.scan.app.monitor.entity.IndexEntity; -import org.apache.linkis.monitor.scan.app.monitor.until.HttpsUntils; -import org.apache.linkis.monitor.scan.constants.Constants; -import org.apache.linkis.monitor.scan.utils.alert.AlertDesc; -import org.apache.linkis.monitor.scan.utils.alert.ims.MonitorAlertUtils; -import org.apache.linkis.monitor.scan.utils.alert.ims.PooledImsAlertUtils; +import org.apache.linkis.governance.common.conf.GovernanceCommonConf; +import org.apache.linkis.monitor.config.MonitorConfig; +import org.apache.linkis.monitor.entity.IndexEntity; +import org.apache.linkis.monitor.until.HttpsUntils; +import org.apache.linkis.monitor.constants.Constants; +import org.apache.linkis.monitor.utils.alert.AlertDesc; +import org.apache.linkis.monitor.utils.alert.ims.MonitorAlertUtils; +import org.apache.linkis.monitor.utils.alert.ims.PooledImsAlertUtils; import org.apache.linkis.rpc.Sender; import org.apache.linkis.server.BDPJettyServerHelper; @@ -49,21 +50,21 @@ */ @Component @PropertySource(value = "classpath:linkis-et-monitor.properties", encoding = "UTF-8") -public class TaskMonitor { +public class EntranceTaskMonitor { - private static final Logger logger = LoggerFactory.getLogger(TaskMonitor.class); + private static final Logger logger = LoggerFactory.getLogger(EntranceTaskMonitor.class); private static final String ENTRANCE_RUNNING_TASK = "entrance_running_task"; private static final String ENTRANCE_QUEUED_TASK = "entrance_queued_task"; - @Scheduled(cron = "${linkis.monitor.entranceTask.cron}") + @Scheduled(cron = "${linkis.monitor.entrance.task.cron}") public void entranceTask() throws IOException { List> userlist = new ArrayList<>(); String value = MonitorConfig.ENTRANCE_TASK_USERLIST.getValue(); if (StringUtils.isNotBlank(value)) { userlist = BDPJettyServerHelper.gson().fromJson(value, ArrayList.class); } - // 用户监控 + userlist.forEach( entranceEntity -> { Map data = new HashMap<>(); @@ -110,7 +111,6 @@ public void entranceTask() throws IOException { } int usertotalTask = MonitorConfig.ENTRANCE_TASK_USERTOTAL.getValue(); if (total.intValue() > usertotalTask) { - // 触发告警 用户任务总数满 parms.put("$tasktotal", String.valueOf(usertotalTask)); Map ecmResourceAlerts = MonitorAlertUtils.getAlerts(Constants.ALERT_RESOURCE_MONITOR(), parms); @@ -142,21 +142,18 @@ public void entranceTask() throws IOException { PooledImsAlertUtils.addAlert(ecmResourceAlerts.get("12009")); } else if (total.intValue() >= linkisTotalMinor) { - // 触发告警Minor parms.put("$taskminor", String.valueOf(linkisTotalMinor)); logger.info("TaskMonitor parms {}:", parms); Map ecmResourceAlerts = MonitorAlertUtils.getAlerts(Constants.ALERT_RESOURCE_MONITOR(), parms); PooledImsAlertUtils.addAlert(ecmResourceAlerts.get("12008")); } - // 指标上报 resourceSendToIms(); } public static void resourceSendToIms() { - // 获取所有的entrance实例，逐个上送IMS ServiceInstance[] instances = - Sender.getInstances(Constants.DIRTY_DATA_ENTRANCE_APPLICATIONNAME()); + Sender.getInstances(GovernanceCommonConf.ENTRANCE_SERVICE_NAME().getValue()); if (null != instances) { for (ServiceInstance instance : instances) { String serviceInstance = instance.getInstance(); diff --git a/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/monitor/scheduled/JobHistoryClear.java b/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scheduled/JobHistoryClear.java similarity index 84% rename from linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/monitor/scheduled/JobHistoryClear.java rename to linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scheduled/JobHistoryClear.java index ae14770420..346ca9cb06 100644 --- a/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/monitor/scheduled/JobHistoryClear.java +++ b/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scheduled/JobHistoryClear.java @@ -15,11 +15,11 @@ * limitations under the License. */ -package org.apache.linkis.monitor.scan.app.monitor.scheduled; +package org.apache.linkis.monitor.scheduled; -import org.apache.linkis.monitor.scan.app.monitor.config.MonitorConfig; -import org.apache.linkis.monitor.scan.app.monitor.until.ThreadUtils; -import org.apache.linkis.monitor.scan.utils.log.LogUtils; +import org.apache.linkis.monitor.config.MonitorConfig; +import org.apache.linkis.monitor.until.ThreadUtils; +import org.apache.linkis.monitor.utils.log.LogUtils; import org.springframework.context.annotation.PropertySource; import org.springframework.scheduling.annotation.Scheduled; @@ -37,7 +37,7 @@ public class JobHistoryClear { private static final Logger logger = LogUtils.stdOutLogger(); - @Scheduled(cron = "${linkis.monitor.clear.historyTask.cron}") + @Scheduled(cron = "${linkis.monitor.clear.history.task.cron}") public void historyTaskClear() { logger.info("Start to clear_history_task shell"); List cmdlist = new ArrayList<>(); diff --git a/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/monitor/scheduled/JobHistoryMonitor.java b/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scheduled/JobHistoryMonitor.java similarity index 77% rename from linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/monitor/scheduled/JobHistoryMonitor.java rename to linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scheduled/JobHistoryMonitor.java index fbca945a8d..93dd028c3c 100644 --- a/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/monitor/scheduled/JobHistoryMonitor.java +++ b/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scheduled/JobHistoryMonitor.java @@ -15,31 +15,31 @@ * limitations under the License. */ -package org.apache.linkis.monitor.scan.app.monitor.scheduled; +package org.apache.linkis.monitor.scheduled; +import org.apache.linkis.monitor.jobhistory.JobHistoryDataFetcher; +import org.apache.linkis.monitor.config.MonitorConfig; +import org.apache.linkis.monitor.until.CacheUtils; import org.apache.linkis.monitor.scan.app.factory.MapperFactory; -import org.apache.linkis.monitor.scan.app.jobhistory.JobHistoryDataFetcher; -import org.apache.linkis.monitor.scan.app.jobhistory.errorcode.JobHistoryErrCodeRule; -import org.apache.linkis.monitor.scan.app.jobhistory.errorcode.JobHistoryErrorCodeAlertSender; -import org.apache.linkis.monitor.scan.app.jobhistory.jobtime.JobTimeExceedAlertSender; -import org.apache.linkis.monitor.scan.app.jobhistory.jobtime.JobTimeExceedRule; -import org.apache.linkis.monitor.scan.app.jobhistory.labels.JobHistoryLabelsAlertSender; -import org.apache.linkis.monitor.scan.app.jobhistory.labels.JobHistoryLabelsRule; -import org.apache.linkis.monitor.scan.app.jobhistory.runtime.CommonJobRunTimeRule; -import org.apache.linkis.monitor.scan.app.jobhistory.runtime.CommonRunTimeAlertSender; -import org.apache.linkis.monitor.scan.app.jobhistory.runtime.JobHistoryRunTimeAlertSender; -import org.apache.linkis.monitor.scan.app.jobhistory.runtime.JobHistoryRunTimeRule; -import org.apache.linkis.monitor.scan.app.monitor.config.MonitorConfig; -import org.apache.linkis.monitor.scan.app.monitor.until.CacheUtils; -import org.apache.linkis.monitor.scan.constants.Constants; -import org.apache.linkis.monitor.scan.core.pac.DataFetcher; -import org.apache.linkis.monitor.scan.core.scanner.AnomalyScanner; -import org.apache.linkis.monitor.scan.core.scanner.DefaultScanner; -import org.apache.linkis.monitor.scan.utils.alert.AlertDesc; -import org.apache.linkis.monitor.scan.utils.alert.ims.ImsAlertDesc; -import org.apache.linkis.monitor.scan.utils.alert.ims.MonitorAlertUtils; -import org.apache.linkis.monitor.scan.utils.alert.ims.UserLabelAlertUtils; -import org.apache.linkis.monitor.scan.utils.log.LogUtils; +import org.apache.linkis.monitor.jobhistory.errorcode.JobHistoryErrCodeRule; +import org.apache.linkis.monitor.jobhistory.errorcode.JobHistoryErrorCodeAlertSender; +import org.apache.linkis.monitor.jobhistory.jobtime.JobTimeExceedAlertSender; +import org.apache.linkis.monitor.jobhistory.jobtime.JobTimeExceedRule; +import org.apache.linkis.monitor.jobhistory.labels.JobHistoryLabelsAlertSender; +import org.apache.linkis.monitor.jobhistory.labels.JobHistoryLabelsRule; +import org.apache.linkis.monitor.jobhistory.runtime.CommonJobRunTimeRule; +import org.apache.linkis.monitor.jobhistory.runtime.CommonRunTimeAlertSender; +import org.apache.linkis.monitor.jobhistory.runtime.JobHistoryRunTimeAlertSender; +import org.apache.linkis.monitor.jobhistory.runtime.JobHistoryRunTimeRule; +import org.apache.linkis.monitor.constants.Constants; +import org.apache.linkis.monitor.core.pac.DataFetcher; +import org.apache.linkis.monitor.core.scanner.AnomalyScanner; +import org.apache.linkis.monitor.core.scanner.DefaultScanner; +import org.apache.linkis.monitor.utils.alert.AlertDesc; +import org.apache.linkis.monitor.utils.alert.ims.ImsAlertDesc; +import org.apache.linkis.monitor.utils.alert.ims.MonitorAlertUtils; +import org.apache.linkis.monitor.utils.alert.ims.UserLabelAlertUtils; +import org.apache.linkis.monitor.utils.log.LogUtils; import org.springframework.context.annotation.PropertySource; import org.springframework.scheduling.annotation.Scheduled; @@ -50,14 +50,18 @@ import org.slf4j.Logger; /** - * * jobHistory monitor 1.errorCode: Monitor the error code 2.userLabel: tenant label monitoring, - * scan the execution data within the first 20 minutes, and judge the labels field of the data + * * jobHistory monitor + * 1.errorCode: Monitor the error code + * 2.userLabel: tenant label monitoring,scan the execution data within the first 20 minutes, and judge the labels field of the data + * * 3.jobResultRunTime: Scan the execution data within the first 20 minutes, and judge the completed * tasks. If the parm field in the jobhistory contains (task.notification.conditions) and the result * of executing the task is (Succeed, Failed, Canceled, Timeout, ALL) any one of them, an alarm is * triggered and the result of the job is that it has ended. All three are indispensable + * * 4.jobResultRunTimeForDSS: Scan the execution data within the first 20 minutes, scan the tasks * that have been marked for notification, if the task has ended, a notification will be initiated + * * 5.jobHistoryUnfinishedScan: monitor the status of the execution task, scan the data outside 12 * hours and within 24 hours */ @@ -68,14 +72,13 @@ public class JobHistoryMonitor { private static final Logger logger = LogUtils.stdOutLogger(); private static final long backtrackNum = 1000000L; - /** Scan tasks completed within 20 minutes */ - @Scheduled(cron = "0 0/10 * * * ?") + @Scheduled(cron = "${linkis.monitor.jobHistory.finished.cron}") public void jobHistoryFinishedScan() { - long intervalMs = 20 * 60 * 1000; // 20分钟 + long intervalMs = 20 * 60 * 1000; long maxIntervalMs = Constants.ERRORCODE_MAX_INTERVALS_SECONDS() * 1000; long endTime = System.currentTimeMillis(); long startTime = endTime - intervalMs; - long realIntervals = endTime - startTime < maxIntervalMs ? endTime - startTime : maxIntervalMs; + long realIntervals = Math.min(endTime - startTime, maxIntervalMs); AnomalyScanner scanner = new DefaultScanner(); boolean shouldStart = false; long id; @@ -91,7 +94,7 @@ public void jobHistoryFinishedScan() { id = CacheUtils.cacheBuilder.getIfPresent("jobHistoryId"); } List fetchers = generateFetchersfortime(startTime, endTime, id, "updated_time"); - if (fetchers == null) { + if (fetchers.isEmpty()) { logger.warn("generated 0 dataFetchers, plz check input"); return; } @@ -174,22 +177,21 @@ public void jobHistoryFinishedScan() { run(scanner, fetchers, shouldStart); } - /** * Whether scanning data within 12 hours has timed out */ - @Scheduled(cron = "${linkis.monitor.jobHistory.ScanTask.cron}") + @Scheduled(cron = "${linkis.monitor.jobHistory.timeout.cron}") public void jobHistoryUnfinishedScan() { long id = Optional.ofNullable(CacheUtils.cacheBuilder.getIfPresent("jobhistoryScan")) .orElse(MonitorConfig.JOB_HISTORY_TIME_EXCEED.getValue()); - long intervalMs = Constants.ERRORCODE_SCAN_INTERVALS_SECONDS() * 1000; + long intervalMs = Constants.TIMEOUT_INTERVALS_SECONDS() * 1000; long maxIntervalMs = Constants.ERRORCODE_MAX_INTERVALS_SECONDS() * 1000; long endTime = System.currentTimeMillis(); long startTime = endTime - intervalMs; - long realIntervals = endTime - startTime < maxIntervalMs ? endTime - startTime : maxIntervalMs; + long realIntervals = Math.min(endTime - startTime, maxIntervalMs); AnomalyScanner scanner = new DefaultScanner(); boolean shouldStart = false; List fetchers = generateFetchers(startTime, endTime, maxIntervalMs, id, "created_time"); - if (fetchers == null) { + if (fetchers.isEmpty()) { logger.warn("generated 0 dataFetchers, plz check input"); return; } @@ -215,7 +217,6 @@ public static void run(AnomalyScanner scanner, List fetchers, Boole if (shouldStart) { scanner.addDataFetchers(fetchers); scanner.run(); - // scanner.shutdown(); // wait all alert to be send } } @@ -225,15 +226,15 @@ private static List generateFetchers( long pe = endTime; long ps; while (pe > startTime) { - ps = pe - maxIntervalMs > startTime ? pe - maxIntervalMs : startTime; + ps = Math.max(pe - maxIntervalMs, startTime); String[] fetcherArgs = new String[] {String.valueOf(ps), String.valueOf(pe), String.valueOf(id), timeType}; ret.add(new JobHistoryDataFetcher(fetcherArgs, MapperFactory.getJobHistoryMapper())); logger.info( "Generated dataFetcher for startTime: " - + new Date(ps).toString() + + new Date(ps) + ". EndTime: " - + new Date(pe).toString()); + + new Date(pe)); pe = pe - maxIntervalMs; } return ret; @@ -249,9 +250,9 @@ private static List generateFetchersfortime( fetchers.add(new JobHistoryDataFetcher(fetcherArgs, MapperFactory.getJobHistoryMapper())); logger.info( "Generated dataFetcher for startTime: " - + new Date(startTime).toString() + + new Date(startTime) + ". EndTime: " - + new Date(endTime).toString()); + + new Date(endTime)); return fetchers; } diff --git a/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/monitor/scheduled/ResourceMonitor.java b/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scheduled/ResourceMonitor.java similarity index 90% rename from linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/monitor/scheduled/ResourceMonitor.java rename to linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scheduled/ResourceMonitor.java index 459aaf70ff..0bc8fe7024 100644 --- a/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/monitor/scheduled/ResourceMonitor.java +++ b/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scheduled/ResourceMonitor.java @@ -15,16 +15,16 @@ * limitations under the License. */ -package org.apache.linkis.monitor.scan.app.monitor.scheduled; +package org.apache.linkis.monitor.scheduled; import org.apache.linkis.common.utils.ByteTimeUtils; -import org.apache.linkis.monitor.scan.app.monitor.config.MonitorConfig; -import org.apache.linkis.monitor.scan.app.monitor.entity.IndexEntity; -import org.apache.linkis.monitor.scan.app.monitor.until.HttpsUntils; -import org.apache.linkis.monitor.scan.constants.Constants; -import org.apache.linkis.monitor.scan.utils.alert.AlertDesc; -import org.apache.linkis.monitor.scan.utils.alert.ims.MonitorAlertUtils; -import org.apache.linkis.monitor.scan.utils.alert.ims.PooledImsAlertUtils; +import org.apache.linkis.monitor.config.MonitorConfig; +import org.apache.linkis.monitor.entity.IndexEntity; +import org.apache.linkis.monitor.until.HttpsUntils; +import org.apache.linkis.monitor.constants.Constants; +import org.apache.linkis.monitor.utils.alert.AlertDesc; +import org.apache.linkis.monitor.utils.alert.ims.MonitorAlertUtils; +import org.apache.linkis.monitor.utils.alert.ims.PooledImsAlertUtils; import org.apache.commons.collections.MapUtils; import org.apache.commons.lang3.StringUtils; @@ -48,7 +48,7 @@ public class ResourceMonitor { private static final Logger logger = LoggerFactory.getLogger(ResourceMonitor.class); - @Scheduled(cron = "${linkis.monitor.ecmResourceTask.cron}") + @Scheduled(cron = "${linkis.monitor.ecm.resource.cron}") public void ecmResourceTask() { Map resultmap = null; AtomicReference tenant = new AtomicReference<>("租户标签：公共资源"); @@ -105,12 +105,10 @@ public void ecmResourceTask() { if (((memorydouble) <= majorValue) || ((coresdouble) <= majorValue) || ((instancedouble) <= majorValue)) { - // major告警 major.add(emNodeVo.get("instance").toString()); } else if (((memorydouble) < minorValue) || ((coresdouble) < minorValue) || ((instancedouble) < minorValue)) { - // minor告警 minor.add(emNodeVo.get("instance").toString()); } HashMap replaceParm = new HashMap<>(); @@ -129,11 +127,9 @@ public void ecmResourceTask() { MonitorAlertUtils.getAlerts(Constants.ALERT_RESOURCE_MONITOR(), replaceParm); PooledImsAlertUtils.addAlert(ecmResourceAlerts.get("12003")); } - // ECM资源占比上报 resourceSendToIms( coresdouble, memorydouble, instancedouble, HttpsUntils.localHost, "USED"); }); - // ECM 剩余资源总数上报 resourceSendToIms( totalCores.get(), totalMemory.get(), totalInstance.get(), HttpsUntils.localHost, "TOTAL"); } diff --git a/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/monitor/scheduled/TaskLogClear.java b/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scheduled/TaskLogClear.java similarity index 87% rename from linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/monitor/scheduled/TaskLogClear.java rename to linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scheduled/TaskLogClear.java index ae7e1bc21d..6def756982 100644 --- a/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/monitor/scheduled/TaskLogClear.java +++ b/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scheduled/TaskLogClear.java @@ -15,11 +15,11 @@ * limitations under the License. */ -package org.apache.linkis.monitor.scan.app.monitor.scheduled; +package org.apache.linkis.monitor.scheduled; -import org.apache.linkis.monitor.scan.app.monitor.config.MonitorConfig; -import org.apache.linkis.monitor.scan.app.monitor.until.ThreadUtils; -import org.apache.linkis.monitor.scan.utils.log.LogUtils; +import org.apache.linkis.monitor.config.MonitorConfig; +import org.apache.linkis.monitor.until.ThreadUtils; +import org.apache.linkis.monitor.utils.log.LogUtils; import org.springframework.context.annotation.PropertySource; import org.springframework.scheduling.annotation.Scheduled; diff --git a/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/monitor/scheduled/UserModeMonitor.java b/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scheduled/UserModeMonitor.java similarity index 94% rename from linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/monitor/scheduled/UserModeMonitor.java rename to linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scheduled/UserModeMonitor.java index c642a24762..e0a7f9f0af 100644 --- a/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/monitor/scheduled/UserModeMonitor.java +++ b/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scheduled/UserModeMonitor.java @@ -15,18 +15,18 @@ * limitations under the License. */ -package org.apache.linkis.monitor.scan.app.monitor.scheduled; +package org.apache.linkis.monitor.scheduled; import org.apache.linkis.common.utils.Utils; import org.apache.linkis.governance.common.entity.task.RequestPersistTask; import org.apache.linkis.httpclient.dws.config.DWSClientConfig; import org.apache.linkis.manager.label.constant.LabelKeyConstant; -import org.apache.linkis.monitor.scan.app.monitor.config.MonitorConfig; -import org.apache.linkis.monitor.scan.app.monitor.until.HttpsUntils; -import org.apache.linkis.monitor.scan.constants.Constants; -import org.apache.linkis.monitor.scan.utils.alert.AlertDesc; -import org.apache.linkis.monitor.scan.utils.alert.ims.MonitorAlertUtils; -import org.apache.linkis.monitor.scan.utils.alert.ims.PooledImsAlertUtils; +import org.apache.linkis.monitor.config.MonitorConfig; +import org.apache.linkis.monitor.until.HttpsUntils; +import org.apache.linkis.monitor.constants.Constants; +import org.apache.linkis.monitor.utils.alert.AlertDesc; +import org.apache.linkis.monitor.utils.alert.ims.MonitorAlertUtils; +import org.apache.linkis.monitor.utils.alert.ims.PooledImsAlertUtils; import org.apache.linkis.server.BDPJettyServerHelper; import org.apache.linkis.ujes.client.UJESClient; import org.apache.linkis.ujes.client.UJESClientImpl; @@ -60,7 +60,7 @@ public class UserModeMonitor { private static final UJESClient client = new UJESClientImpl(clientConfig); - @Scheduled(cron = "${linkis.monitor.user.cron}") + @Scheduled(cron = "${linkis.monitor.user.mode.cron}") public void job() { Optional.ofNullable(MonitorConfig.USER_MODE_ENGINE.getValue()) .ifPresent( diff --git a/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/monitor/until/CacheUtils.java b/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/until/CacheUtils.java similarity index 95% rename from linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/monitor/until/CacheUtils.java rename to linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/until/CacheUtils.java index 258d81183c..a768fde555 100644 --- a/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/monitor/until/CacheUtils.java +++ b/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/until/CacheUtils.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.linkis.monitor.scan.app.monitor.until; +package org.apache.linkis.monitor.until; import java.util.concurrent.TimeUnit; diff --git a/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/monitor/until/HttpsUntils.java b/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/until/HttpsUntils.java similarity index 82% rename from linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/monitor/until/HttpsUntils.java rename to linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/until/HttpsUntils.java index 34e12d175d..fd3f584b34 100644 --- a/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/monitor/until/HttpsUntils.java +++ b/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/until/HttpsUntils.java @@ -15,22 +15,21 @@ * limitations under the License. */ -package org.apache.linkis.monitor.scan.app.monitor.until; +package org.apache.linkis.monitor.until; import org.apache.linkis.common.conf.Configuration; import org.apache.linkis.common.utils.Utils; import org.apache.linkis.httpclient.dws.authentication.TokenAuthenticationStrategy; import org.apache.linkis.httpclient.dws.config.DWSClientConfig; import org.apache.linkis.httpclient.dws.config.DWSClientConfigBuilder; -import org.apache.linkis.monitor.scan.app.monitor.config.MonitorConfig; -import org.apache.linkis.monitor.scan.app.monitor.entity.ChatbotEntity; -import org.apache.linkis.monitor.scan.app.monitor.entity.IndexEntity; -import org.apache.linkis.monitor.scan.client.MonitorHTTPClient; -import org.apache.linkis.monitor.scan.client.MonitorHTTPClientClientImpl; -import org.apache.linkis.monitor.scan.constants.Constants; -import org.apache.linkis.monitor.scan.request.EmsListAction; -import org.apache.linkis.monitor.scan.request.EntranceTaskAction; -import org.apache.linkis.monitor.scan.response.EntranceTaskResult; +import org.apache.linkis.monitor.entity.IndexEntity; +import org.apache.linkis.monitor.config.MonitorConfig; +import org.apache.linkis.monitor.client.MonitorHTTPClient; +import org.apache.linkis.monitor.client.MonitorHTTPClientClientImpl; +import org.apache.linkis.monitor.constants.Constants; +import org.apache.linkis.monitor.request.EmsListAction; +import org.apache.linkis.monitor.request.EntranceTaskAction; +import org.apache.linkis.monitor.response.EntranceTaskResult; import org.apache.linkis.server.BDPJettyServerHelper; import org.apache.linkis.ujes.client.response.EmsListResult; @@ -169,18 +168,4 @@ public static void sendIndex(List list) throws IOException { logger.info("send index response :{}", map); Assert.isTrue(!"0".equals(map.get("resultCode")), map.get("resultMsg")); } - - public static void sendChatbot(ChatbotEntity chatbotEntity) throws IOException { - String json = BDPJettyServerHelper.gson().toJson(chatbotEntity); - StringEntity entity = - new StringEntity( - json, ContentType.create(ContentType.APPLICATION_JSON.getMimeType(), "UTF-8")); - entity.setContentEncoding("UTF-8"); - HttpPost httpPost = new HttpPost(MonitorConfig.CHATBOT_URL.getValue()); - httpPost.setConfig(RequestConfig.DEFAULT); - httpPost.setEntity(entity); - CloseableHttpResponse execute = HttpClients.createDefault().execute(httpPost); - String responseStr = EntityUtils.toString(execute.getEntity(), "UTF-8"); - Map map = BDPJettyServerHelper.gson().fromJson(responseStr, Map.class); - } } diff --git a/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/monitor/until/ThreadUtils.java b/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/until/ThreadUtils.java similarity index 80% rename from linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/monitor/until/ThreadUtils.java rename to linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/until/ThreadUtils.java index 5a099e7b65..15a2626379 100644 --- a/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/scan/app/monitor/until/ThreadUtils.java +++ b/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/until/ThreadUtils.java @@ -15,15 +15,15 @@ * limitations under the License. */ -package org.apache.linkis.monitor.scan.app.monitor.until; +package org.apache.linkis.monitor.until; import org.apache.linkis.common.utils.Utils; -import org.apache.linkis.monitor.scan.app.monitor.config.MonitorConfig; -import org.apache.linkis.monitor.scan.constants.Constants; -import org.apache.linkis.monitor.scan.utils.alert.AlertDesc; -import org.apache.linkis.monitor.scan.utils.alert.ims.MonitorAlertUtils; -import org.apache.linkis.monitor.scan.utils.alert.ims.PooledImsAlertUtils; -import org.apache.linkis.monitor.scan.utils.log.LogUtils; +import org.apache.linkis.monitor.config.MonitorConfig; +import org.apache.linkis.monitor.constants.Constants; +import org.apache.linkis.monitor.utils.alert.AlertDesc; +import org.apache.linkis.monitor.utils.alert.ims.MonitorAlertUtils; +import org.apache.linkis.monitor.utils.alert.ims.PooledImsAlertUtils; +import org.apache.linkis.monitor.utils.log.LogUtils; import org.springframework.context.ApplicationContext; import org.springframework.context.event.ApplicationContextEvent; @@ -53,8 +53,7 @@ public static String run(List cmdList, String shellName) { try { msg = future.get(MonitorConfig.SHELL_TIMEOUT.getValue(), TimeUnit.MINUTES).toString(); } catch (TimeoutException e) { - // 增加告警提示 - logger.info("超时告警 {}", shellName); + logger.info("execute shell time out {}", shellName); HashMap parms = new HashMap<>(); parms.put("$shellName", shellName); Map ecmResourceAlerts = diff --git a/linkis-extensions/linkis-et-monitor/src/main/resources/mapper/common/InsLabelRelationMapper.xml b/linkis-extensions/linkis-et-monitor/src/main/resources/mapper/common/InsLabelRelationMapper.xml index b9c1d19612..6c51f6d0aa 100644 --- a/linkis-extensions/linkis-et-monitor/src/main/resources/mapper/common/InsLabelRelationMapper.xml +++ b/linkis-extensions/linkis-et-monitor/src/main/resources/mapper/common/InsLabelRelationMapper.xml @@ -16,24 +16,24 @@ --> - - + + - + - + - + diff --git a/linkis-extensions/linkis-et-monitor/src/main/resources/mapper/common/InstanceInfoMapper.xml b/linkis-extensions/linkis-et-monitor/src/main/resources/mapper/common/InstanceInfoMapper.xml index c0ab7f2b22..d5309a5f04 100644 --- a/linkis-extensions/linkis-et-monitor/src/main/resources/mapper/common/InstanceInfoMapper.xml +++ b/linkis-extensions/linkis-et-monitor/src/main/resources/mapper/common/InstanceInfoMapper.xml @@ -16,9 +16,9 @@ --> - + - + diff --git a/linkis-extensions/linkis-et-monitor/src/main/resources/mapper/common/InstanceLabelMapper.xml b/linkis-extensions/linkis-et-monitor/src/main/resources/mapper/common/InstanceLabelMapper.xml index d92d999d47..e7c7558017 100644 --- a/linkis-extensions/linkis-et-monitor/src/main/resources/mapper/common/InstanceLabelMapper.xml +++ b/linkis-extensions/linkis-et-monitor/src/main/resources/mapper/common/InstanceLabelMapper.xml @@ -16,9 +16,9 @@ --> - + - + diff --git a/linkis-extensions/linkis-et-monitor/src/main/resources/mapper/common/JobHistoryMapper.xml b/linkis-extensions/linkis-et-monitor/src/main/resources/mapper/common/JobHistoryMapper.xml index 70d1f30158..730e8a43da 100644 --- a/linkis-extensions/linkis-et-monitor/src/main/resources/mapper/common/JobHistoryMapper.xml +++ b/linkis-extensions/linkis-et-monitor/src/main/resources/mapper/common/JobHistoryMapper.xml @@ -18,9 +18,9 @@ - + - @@ -49,7 +49,7 @@