diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index f38af2d393..531a1ff88a 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -12,7 +12,7 @@ and session management. ### Related issues/PRs -Related issues: #590 +Related issues: close #590 close #591 Related pr:#591 diff --git a/.github/workflows/integration-test.yml b/.github/workflows/integration-test.yml index ce0b8ceec9..4983edcbbc 100644 --- a/.github/workflows/integration-test.yml +++ b/.github/workflows/integration-test.yml @@ -45,7 +45,7 @@ jobs: TAG: ${{ github.sha }} SKIP_TEST: true HUB: ghcr.io/apache/linkis - LINKIS_VERSION: 1.5.0-SNAPSHOT + LINKIS_VERSION: 1.5.0 steps: - name: Free up disk space run: | diff --git a/.github/workflows/publish-docker.yaml b/.github/workflows/publish-docker.yaml index ec07c2eff3..1e51623e84 100644 --- a/.github/workflows/publish-docker.yaml +++ b/.github/workflows/publish-docker.yaml @@ -34,7 +34,7 @@ jobs: TAG: ${{ github.sha }} SKIP_TEST: true HUB: ghcr.io/apache/linkis - LINKIS_VERSION: 1.5.0-SNAPSHOT + LINKIS_VERSION: 1.5.0 steps: - name: Checkout uses: actions/checkout@v2 diff --git a/.github/workflows/publish-snapshot.yml b/.github/workflows/publish-snapshot.yml index 8bb65c90f4..dfcee9e221 100644 --- a/.github/workflows/publish-snapshot.yml +++ b/.github/workflows/publish-snapshot.yml @@ -29,7 +29,7 @@ jobs: fail-fast: false matrix: branch: - - dev-1.4.0 + - dev-1.5.0 steps: - name: Checkout repository uses: actions/checkout@v2 diff --git a/linkis-commons/linkis-common/src/test/java/org/apache/linkis/common/exception/ExceptionManagerTest.java b/linkis-commons/linkis-common/src/test/java/org/apache/linkis/common/exception/ExceptionManagerTest.java index d45bebc125..839a34f859 100644 --- a/linkis-commons/linkis-common/src/test/java/org/apache/linkis/common/exception/ExceptionManagerTest.java +++ b/linkis-commons/linkis-common/src/test/java/org/apache/linkis/common/exception/ExceptionManagerTest.java @@ -21,6 +21,7 @@ import java.util.HashMap; import java.util.Map; +import java.util.TreeMap; import org.junit.jupiter.api.Test; @@ -40,7 +41,7 @@ void testGenerateException() { + "null"); assertEquals(errorException.getClass(), ExceptionManager.generateException(null).getClass()); assertEquals(errorException.toString(), ExceptionManager.generateException(null).toString()); - Map map = new HashMap<>(); + Map map = new TreeMap<>(); map.put("level", null); map.put("errCode", 1); map.put("desc", "test"); diff --git a/linkis-commons/linkis-module/pom.xml b/linkis-commons/linkis-module/pom.xml index c5b247b9be..328480f25b 100644 --- a/linkis-commons/linkis-module/pom.xml +++ b/linkis-commons/linkis-module/pom.xml @@ -64,10 +64,6 @@ - - org.springframework.cloud - spring-cloud-starter-netflix-eureka-client - org.springframework.boot @@ -281,4 +277,80 @@ + + + eureka + + true + + discovery + eureka + + + + + org.springframework.cloud + spring-cloud-starter-netflix-eureka-client + + + + + nacos + + + discovery + nacos + + + + + com.alibaba.cloud + spring-cloud-starter-alibaba-nacos-discovery + + + org.springframework.boot + * + + + org.springframework.cloud + spring-cloud-commons + + + org.springframework.cloud + spring-cloud-context + + + org.springframework.boot + spring-boot-starter + + + com.fasterxml.jackson.core + jackson-core + + + com.fasterxml.jackson.core + jackson-databind + + + com.google.code.findbugs + jsr305 + + + org.yaml + snakeyaml + + + io.prometheus + simpleclient + + + com.google.guava + guava + + + + + + + diff --git a/linkis-commons/linkis-storage/pom.xml b/linkis-commons/linkis-storage/pom.xml index def795ebd8..2f1cdb44bd 100644 --- a/linkis-commons/linkis-storage/pom.xml +++ b/linkis-commons/linkis-storage/pom.xml @@ -99,6 +99,52 @@ aws-java-sdk-s3 1.12.261 + + + org.apache.parquet + parquet-avro + ${parquet-avro.version} + ${storage.parquet.scope} + + + org.apache.hadoop + hadoop-mapreduce-client-core + ${hadoop.version} + ${storage.parquet.scope} + + + log4j + log4j + + + org.slf4j + slf4j-log4j12 + + + + ch.qos.reload4j + reload4j + + + org.slf4j + slf4j-reload4j + + + + + org.apache.orc + orc-core + ${orc-core.version} + nohive + ${storage.orc.scope} + + + org.apache.hive + hive-storage-api + + + + diff --git a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/conf/LinkisStorageConf.java b/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/conf/LinkisStorageConf.java index 74950c15fe..7fb5c1b4ac 100644 --- a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/conf/LinkisStorageConf.java +++ b/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/conf/LinkisStorageConf.java @@ -25,6 +25,16 @@ public class LinkisStorageConf { private static final Object CONF_LOCK = new Object(); + public static final String DOLPHIN = "dolphin"; + + public static final String PARQUET = "parquet"; + + public static final String PARQUET_FILE_SUFFIX = ".parquet"; + + public static final String ORC = "orc"; + + public static final String ORC_FILE_SUFFIX = ".orc"; + public static final String HDFS_FILE_SYSTEM_REST_ERRS = CommonVars.apply( "wds.linkis.hdfs.rest.errs", @@ -34,12 +44,19 @@ public class LinkisStorageConf { public static final String ROW_BYTE_MAX_LEN_STR = CommonVars.apply("wds.linkis.resultset.row.max.str", "2m").getValue(); + public static final String ENGINE_RESULT_TYPE = + CommonVars.apply("linkis.engine.resultSet.type", DOLPHIN, "Result type").getValue(); + public static final long ROW_BYTE_MAX_LEN = ByteTimeUtils.byteStringAsBytes(ROW_BYTE_MAX_LEN_STR); public static final String FILE_TYPE = CommonVars.apply( "wds.linkis.storage.file.type", - "dolphin,sql,scala,py,hql,python,out,log,text,txt,sh,jdbc,ngql,psql,fql,tsql") + "dolphin,sql,scala,py,hql,python,out,log,text,txt,sh,jdbc,ngql,psql,fql,tsql" + + "," + + PARQUET + + "," + + ORC) .getValue(); private static volatile String[] fileTypeArr = null; diff --git a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/exception/StorageErrorCode.java b/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/exception/StorageErrorCode.java index fad0d83a12..7a8452141b 100644 --- a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/exception/StorageErrorCode.java +++ b/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/exception/StorageErrorCode.java @@ -23,7 +23,13 @@ public enum StorageErrorCode { FS_NOT_INIT(53001, "please init first"), INCONSISTENT_DATA(53001, "Inconsistent row data read,read %s,need rowLen %s"), - FS_OOM(53002, "OOM occurred while reading the file"); + FS_OOM(53002, "OOM occurred while reading the file"), + + FS_ERROR(53003, "Failed to operation fs"), + + READ_PARQUET_FAILED(53004, "Failed to read parquet file"), + + READ_ORC_FAILED(53005, "Failed to read orc file"); StorageErrorCode(int errorCode, String message) { this.code = errorCode; diff --git a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/exception/StorageReadException.java b/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/exception/StorageReadException.java new file mode 100644 index 0000000000..dedad1140c --- /dev/null +++ b/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/exception/StorageReadException.java @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.storage.exception; + +import org.apache.linkis.common.exception.ErrorException; + +public class StorageReadException extends ErrorException { + + public StorageReadException(int errCode, String desc) { + super(errCode, desc); + } + + public StorageReadException(int errCode, String desc, Throwable t) { + super(errCode, desc); + initCause(t); + } + + public StorageReadException(int errCode, String desc, String ip, int port, String serviceKind) { + super(errCode, desc, ip, port, serviceKind); + } +} diff --git a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/fs/impl/S3FileSystem.java b/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/fs/impl/S3FileSystem.java index b8f6401b11..026a6f9e0e 100644 --- a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/fs/impl/S3FileSystem.java +++ b/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/fs/impl/S3FileSystem.java @@ -38,10 +38,7 @@ import com.amazonaws.client.builder.AwsClientBuilder; import com.amazonaws.services.s3.AmazonS3; import com.amazonaws.services.s3.AmazonS3ClientBuilder; -import com.amazonaws.services.s3.model.AmazonS3Exception; -import com.amazonaws.services.s3.model.ListObjectsV2Result; -import com.amazonaws.services.s3.model.ObjectMetadata; -import com.amazonaws.services.s3.model.S3ObjectSummary; +import com.amazonaws.services.s3.model.*; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -114,7 +111,7 @@ public FsPath get(String dest) throws IOException { @Override public InputStream read(FsPath dest) throws IOException { try { - return s3Client.getObject(bucket, dest.getPath()).getObjectContent(); + return s3Client.getObject(bucket, buildPrefix(dest.getPath(), false)).getObjectContent(); } catch (AmazonS3Exception e) { throw new IOException("You have not permission to access path " + dest.getPath()); } @@ -123,8 +120,9 @@ public InputStream read(FsPath dest) throws IOException { @Override public OutputStream write(FsPath dest, boolean overwrite) throws IOException { try (InputStream inputStream = read(dest); - OutputStream outputStream = new S3OutputStream(s3Client, bucket, dest.getPath())) { - if (overwrite) { + OutputStream outputStream = + new S3OutputStream(s3Client, bucket, buildPrefix(dest.getPath(), false))) { + if (!overwrite) { IOUtils.copy(inputStream, outputStream); } return outputStream; @@ -164,20 +162,37 @@ public List list(FsPath path) throws IOException { @Override public FsPathListWithError listPathWithError(FsPath path) throws IOException { + return listPathWithError(path, true); + } + + public FsPathListWithError listPathWithError(FsPath path, boolean ignoreInitFile) + throws IOException { + List rtn = new ArrayList<>(); try { if (!StringUtils.isEmpty(path.getPath())) { - ListObjectsV2Result listObjectsV2Result = s3Client.listObjectsV2(bucket, path.getPath()); - List s3ObjectSummaries = listObjectsV2Result.getObjectSummaries(); + ListObjectsV2Request listObjectsV2Request = + new ListObjectsV2Request() + .withBucketName(bucket) + .withPrefix(buildPrefix(path.getPath())) + .withDelimiter("/"); + ListObjectsV2Result dirResult = s3Client.listObjectsV2(listObjectsV2Request); + List s3ObjectSummaries = dirResult.getObjectSummaries(); + List commonPrefixes = dirResult.getCommonPrefixes(); if (s3ObjectSummaries != null) { - List rtn = new ArrayList(); - String message = ""; for (S3ObjectSummary summary : s3ObjectSummaries) { - if (isDir(summary, path.getPath()) || isInitFile(summary)) continue; + if (isInitFile(summary) && ignoreInitFile) continue; FsPath newPath = new FsPath(buildPath(summary.getKey())); rtn.add(fillStorageFile(newPath, summary)); } - return new FsPathListWithError(rtn, message); } + if (commonPrefixes != null) { + for (String dir : commonPrefixes) { + FsPath newPath = new FsPath(buildPath(dir)); + newPath.setIsdir(true); + rtn.add(newPath); + } + } + return new FsPathListWithError(rtn, ""); } } catch (AmazonS3Exception e) { throw new IOException("You have not permission to access path " + path.getPath()); @@ -189,8 +204,25 @@ public FsPathListWithError listPathWithError(FsPath path) throws IOException { @Override public boolean exists(FsPath dest) throws IOException { try { - int size = s3Client.listObjectsV2(bucket, dest.getPath()).getObjectSummaries().size(); - return size > 0; + if (new File(dest.getPath()).getName().contains(".")) { + return existsFile(dest); + } + ListObjectsV2Request listObjectsV2Request = new ListObjectsV2Request(); + listObjectsV2Request + .withBucketName(bucket) + .withPrefix(buildPrefix(dest.getPath())) + .withDelimiter("/"); + return s3Client.listObjectsV2(listObjectsV2Request).getObjectSummaries().size() + + s3Client.listObjectsV2(listObjectsV2Request).getCommonPrefixes().size() + > 0; + } catch (AmazonS3Exception e) { + return false; + } + } + + public boolean existsFile(FsPath dest) { + try { + return s3Client.doesObjectExist(bucket, buildPrefix(dest.getPath(), false)); } catch (AmazonS3Exception e) { return false; } @@ -199,7 +231,14 @@ public boolean exists(FsPath dest) throws IOException { @Override public boolean delete(FsPath dest) throws IOException { try { - s3Client.deleteObject(bucket, dest.getPath()); + ListObjectsV2Request listObjectsV2Request = new ListObjectsV2Request(); + listObjectsV2Request.withBucketName(bucket).withPrefix(buildPrefix(dest.getPath(), false)); + ListObjectsV2Result result = s3Client.listObjectsV2(listObjectsV2Request); + String[] keyList = + result.getObjectSummaries().stream().map(S3ObjectSummary::getKey).toArray(String[]::new); + DeleteObjectsRequest deleteObjectsRequest = + new DeleteObjectsRequest("test").withKeys(keyList); + s3Client.deleteObjects(deleteObjectsRequest); return true; } catch (AmazonS3Exception e) { throw new IOException("You have not permission to access path " + dest.getPath()); @@ -209,8 +248,25 @@ public boolean delete(FsPath dest) throws IOException { @Override public boolean renameTo(FsPath oldDest, FsPath newDest) throws IOException { try { - s3Client.copyObject(bucket, oldDest.getPath(), bucket, newDest.getPath()); - s3Client.deleteObject(bucket, oldDest.getPath()); + String newOriginPath = buildPrefix(oldDest.getPath(), false); + String newDestPath = buildPrefix(newDest.getPath(), false); + ListObjectsV2Request listObjectsV2Request = new ListObjectsV2Request(); + listObjectsV2Request.withBucketName(bucket).withPrefix(newOriginPath); + ListObjectsV2Result result = s3Client.listObjectsV2(listObjectsV2Request); + List keyList = + result.getObjectSummaries().stream() + .map(S3ObjectSummary::getKey) + .collect(Collectors.toList()); + List newKeyList = + keyList.stream() + .map(key -> key.replaceFirst(newOriginPath, newDestPath)) + .collect(Collectors.toList()); + for (int i = 0; i < keyList.size(); i++) { + String key = keyList.get(i); + String newKey = newKeyList.get(i); + s3Client.copyObject(bucket, key, bucket, newKey); + s3Client.deleteObject(bucket, key); + } return true; } catch (AmazonS3Exception e) { s3Client.deleteObject(bucket, newDest.getPath()); @@ -225,7 +281,24 @@ public boolean renameTo(FsPath oldDest, FsPath newDest) throws IOException { @Override public boolean copy(String origin, String dest) throws IOException { try { - s3Client.copyObject(bucket, origin, bucket, dest); + String newOrigin = buildPrefix(origin, false); + String newDest = buildPrefix(dest, false); + ListObjectsV2Request listObjectsV2Request = new ListObjectsV2Request(); + listObjectsV2Request.withBucketName(bucket).withPrefix(newOrigin); + ListObjectsV2Result result = s3Client.listObjectsV2(listObjectsV2Request); + List keyList = + result.getObjectSummaries().stream() + .map(S3ObjectSummary::getKey) + .collect(Collectors.toList()); + List newKeyList = + keyList.stream() + .map(key -> key.replaceFirst(newOrigin, newDest)) + .collect(Collectors.toList()); + for (int i = 0; i < keyList.size(); i++) { + String key = keyList.get(i); + String newKey = newKeyList.get(i); + s3Client.copyObject(bucket, key, bucket, newKey); + } return true; } catch (AmazonS3Exception e) { throw new IOException("You have not permission to access path " + origin + " or " + dest); @@ -261,7 +334,10 @@ public boolean mkdirs(FsPath dest) throws IOException { private FsPath fillStorageFile(FsPath fsPath, S3ObjectSummary s3ObjectSummary) { fsPath.setModification_time(s3ObjectSummary.getLastModified().getTime()); - fsPath.setOwner(s3ObjectSummary.getOwner().getDisplayName()); + Owner owner = s3ObjectSummary.getOwner(); + if (owner != null) { + fsPath.setOwner(owner.getDisplayName()); + } try { fsPath.setIsdir(isDir(s3ObjectSummary, fsPath.getParent().getPath())); } catch (Throwable e) { @@ -344,6 +420,22 @@ public String buildPath(String path) { } return StorageUtils.S3_SCHEMA + "/" + path; } + + public String buildPrefix(String path, boolean addTail) { + String res = path; + if (path == null || "".equals(path)) return ""; + if (path.startsWith("/")) { + res = path.replaceFirst("/", ""); + } + if (!path.endsWith("/") && addTail) { + res = res + "/"; + } + return res; + } + + public String buildPrefix(String path) { + return buildPrefix(path, true); + } } class S3OutputStream extends ByteArrayOutputStream { diff --git a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/resultset/DefaultResultSetFactory.java b/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/resultset/DefaultResultSetFactory.java index db78afac29..da51a2f13d 100644 --- a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/resultset/DefaultResultSetFactory.java +++ b/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/resultset/DefaultResultSetFactory.java @@ -23,6 +23,7 @@ import org.apache.linkis.common.io.Record; import org.apache.linkis.common.io.resultset.ResultSet; import org.apache.linkis.storage.FSFactory; +import org.apache.linkis.storage.conf.LinkisStorageConf; import org.apache.linkis.storage.domain.Dolphin; import org.apache.linkis.storage.errorcode.LinkisStorageErrorCodeSummary; import org.apache.linkis.storage.exception.StorageWarnException; @@ -108,7 +109,9 @@ public boolean exists(String resultSetType) { @Override public boolean isResultSetPath(String path) { - return path.endsWith(Dolphin.DOLPHIN_FILE_SUFFIX); + return path.endsWith(Dolphin.DOLPHIN_FILE_SUFFIX) + || path.endsWith(LinkisStorageConf.PARQUET_FILE_SUFFIX) + || path.endsWith(LinkisStorageConf.ORC_FILE_SUFFIX); } @Override @@ -134,15 +137,22 @@ public String[] getResultSetType() { @Override public ResultSet getResultSetByPath(FsPath fsPath, Fs fs) { + ResultSet resultSet = null; try (InputStream inputStream = fs.read(fsPath)) { - String resultSetType = Dolphin.getType(inputStream); - if (StringUtils.isEmpty(resultSetType)) { - throw new StorageWarnException( - THE_FILE_IS_EMPTY.getErrorCode(), - MessageFormat.format(THE_FILE_IS_EMPTY.getErrorDesc(), fsPath.getPath())); + if (fsPath.getPath().endsWith(Dolphin.DOLPHIN_FILE_SUFFIX)) { + String resultSetType = Dolphin.getType(inputStream); + if (StringUtils.isEmpty(resultSetType)) { + throw new StorageWarnException( + THE_FILE_IS_EMPTY.getErrorCode(), + MessageFormat.format(THE_FILE_IS_EMPTY.getErrorDesc(), fsPath.getPath())); + } + // Utils.tryQuietly(fs::close); + resultSet = getResultSetByType(resultSetType); + } else if (fsPath.getPath().endsWith(LinkisStorageConf.PARQUET_FILE_SUFFIX) + || fsPath.getPath().endsWith(LinkisStorageConf.ORC_FILE_SUFFIX)) { + resultSet = getResultSetByType(ResultSetFactory.TABLE_TYPE); } - // Utils.tryQuietly(fs::close); - return getResultSetByType(resultSetType); + return resultSet; } catch (IOException e) { throw new RuntimeException(e); } diff --git a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/resultset/OrcResultSetReader.java b/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/resultset/OrcResultSetReader.java new file mode 100644 index 0000000000..249e326cde --- /dev/null +++ b/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/resultset/OrcResultSetReader.java @@ -0,0 +1,212 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.storage.resultset; + +import org.apache.linkis.common.io.FsPath; +import org.apache.linkis.common.io.MetaData; +import org.apache.linkis.common.io.Record; +import org.apache.linkis.common.io.resultset.ResultSet; +import org.apache.linkis.common.io.resultset.ResultSetReader; +import org.apache.linkis.storage.domain.Column; +import org.apache.linkis.storage.domain.DataType; +import org.apache.linkis.storage.resultset.table.TableMetaData; +import org.apache.linkis.storage.resultset.table.TableRecord; + +import org.apache.commons.io.IOUtils; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.Path; +import org.apache.orc.OrcFile; +import org.apache.orc.Reader; +import org.apache.orc.RecordReader; +import org.apache.orc.TypeDescription; +import org.apache.orc.storage.ql.exec.vector.*; + +import java.io.IOException; +import java.io.InputStream; +import java.util.ArrayList; +import java.util.List; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class OrcResultSetReader + extends ResultSetReader { + + private static final Logger logger = LoggerFactory.getLogger(OrcResultSetReader.class); + + private FsPath fsPath; + + private final ResultSet resultSet; + + private final InputStream inputStream; + + private MetaData metaData; + + private Record row; + + private RecordReader rows; + + private Reader reader; + + public OrcResultSetReader(ResultSet resultSet, InputStream inputStream, FsPath fsPath) + throws IOException { + super(resultSet, inputStream); + this.resultSet = resultSet; + this.inputStream = inputStream; + this.fsPath = fsPath; + this.reader = + OrcFile.createReader( + new Path(fsPath.getPath()), OrcFile.readerOptions(new Configuration())); + this.rows = reader.rows(); + } + + @Override + public MetaData getMetaData() { + if (metaData == null) { + try { + List fieldNames = reader.getSchema().getFieldNames(); + List typeDescriptions = reader.getSchema().getChildren(); + List columnList = new ArrayList<>(); + for (int i = 0; i < fieldNames.size(); i++) { + Column column = + new Column( + fieldNames.get(i), + DataType.toDataType(typeDescriptions.get(i).getCategory().getName()), + ""); + columnList.add(column); + } + + metaData = new TableMetaData(columnList.toArray(new Column[0])); + } catch (Exception e) { + throw new RuntimeException("Failed to read parquet schema", e); + } + } + return metaData; + } + + @Override + public int skip(int recordNum) throws IOException { + if (recordNum < 0) return -1; + + for (int i = recordNum; i > 0; i--) { + try { + hasNext(); + } catch (Throwable t) { + return recordNum - i; + } + } + return recordNum; + } + + @Override + public long getPosition() throws IOException { + throw new UnsupportedOperationException("Storeage Unsupported type: getPosition"); + } + + @Override + public long available() throws IOException { + throw new UnsupportedOperationException("Storeage Unsupported type: available"); + } + + @Override + public boolean hasNext() throws IOException { + if (metaData == null) getMetaData(); + if (rows == null) return false; + VectorizedRowBatch batch = + reader.getSchema().createRowBatch(TypeDescription.RowBatchVersion.ORIGINAL, 1); + TableMetaData tableMetaData = (TableMetaData) metaData; + + if (rows.nextBatch(batch)) { + int rowNum = 0; + Object[] rowData = new Object[tableMetaData.getColumns().length]; + for (int i = 0; i < batch.numCols; i++) { + ColumnVector columnVector = batch.cols[i]; + if (columnVector instanceof BytesColumnVector) { + BytesColumnVector vector = (BytesColumnVector) columnVector; + rowData[i] = vector.toString(rowNum); + } else if (columnVector instanceof Decimal64ColumnVector) { + Decimal64ColumnVector vector = (Decimal64ColumnVector) columnVector; + rowData[i] = vector.vector[rowNum]; + } else if (columnVector instanceof DecimalColumnVector) { + DecimalColumnVector vector = (DecimalColumnVector) columnVector; + rowData[i] = vector.vector[rowNum]; + } else if (columnVector instanceof DoubleColumnVector) { + DoubleColumnVector vector = (DoubleColumnVector) columnVector; + rowData[i] = vector.vector[rowNum]; + } else if (columnVector instanceof ListColumnVector) { + ListColumnVector vector = (ListColumnVector) columnVector; + StringBuilder builder = new StringBuilder(); + vector.stringifyValue(builder, rowNum); + rowData[i] = builder.toString(); + } else if (columnVector instanceof IntervalDayTimeColumnVector) { + IntervalDayTimeColumnVector vector = (IntervalDayTimeColumnVector) columnVector; + StringBuilder builder = new StringBuilder(); + vector.stringifyValue(builder, rowNum); + rowData[i] = builder.toString(); + } else if (columnVector instanceof LongColumnVector) { + LongColumnVector vector = (LongColumnVector) columnVector; + rowData[i] = vector.vector[rowNum]; + } else if (columnVector instanceof MapColumnVector) { + MapColumnVector vector = (MapColumnVector) columnVector; + StringBuilder builder = new StringBuilder(); + vector.stringifyValue(builder, rowNum); + rowData[i] = builder.toString(); + } else if (columnVector instanceof MultiValuedColumnVector) { + MultiValuedColumnVector vector = (MultiValuedColumnVector) columnVector; + StringBuilder builder = new StringBuilder(); + vector.stringifyValue(builder, rowNum); + rowData[i] = builder.toString(); + } else if (columnVector instanceof StructColumnVector) { + StructColumnVector vector = (StructColumnVector) columnVector; + StringBuilder builder = new StringBuilder(); + vector.stringifyValue(builder, rowNum); + rowData[i] = builder.toString(); + } else if (columnVector instanceof TimestampColumnVector) { + TimestampColumnVector vector = (TimestampColumnVector) columnVector; + rowData[i] = vector.time[rowNum]; + } else if (columnVector instanceof UnionColumnVector) { + UnionColumnVector vector = (UnionColumnVector) columnVector; + StringBuilder builder = new StringBuilder(); + vector.stringifyValue(builder, rowNum); + rowData[i] = builder.toString(); + } + } + row = new TableRecord(rowData); + } else { + return false; + } + return row != null; + } + + @Override + public Record getRecord() { + if (metaData == null) throw new RuntimeException("Must read metadata first(必须先读取metadata)"); + if (row == null) { + throw new RuntimeException( + "Can't get the value of the field, maybe the IO stream has been read or has been closed!(拿不到字段的值，也许IO流已读取完毕或已被关闭！)"); + } + return row; + } + + @Override + public void close() throws IOException { + IOUtils.closeQuietly(inputStream); + rows.close(); + reader.close(); + } +} diff --git a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/resultset/OrcResultSetWriter.java b/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/resultset/OrcResultSetWriter.java new file mode 100644 index 0000000000..c4809f6499 --- /dev/null +++ b/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/resultset/OrcResultSetWriter.java @@ -0,0 +1,141 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.storage.resultset; + +import org.apache.linkis.common.io.FsPath; +import org.apache.linkis.common.io.MetaData; +import org.apache.linkis.common.io.Record; +import org.apache.linkis.common.io.resultset.ResultSet; +import org.apache.linkis.common.io.resultset.ResultSetWriter; +import org.apache.linkis.storage.domain.Column; +import org.apache.linkis.storage.resultset.table.TableMetaData; +import org.apache.linkis.storage.resultset.table.TableRecord; +import org.apache.linkis.storage.utils.OrcUtils; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.Path; +import org.apache.orc.CompressionKind; +import org.apache.orc.OrcFile; +import org.apache.orc.TypeDescription; +import org.apache.orc.Writer; +import org.apache.orc.storage.ql.exec.vector.VectorizedRowBatch; + +import java.io.IOException; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class OrcResultSetWriter + extends ResultSetWriter { + private static final Logger logger = LoggerFactory.getLogger(OrcResultSetWriter.class); + + private TypeDescription schema; + + private Writer writer; + + private boolean moveToWriteRow = false; + + private MetaData metaData = null; + + private final FsPath storePath; + + private final long maxCacheSize; + + private final ResultSet resultSet; + + public OrcResultSetWriter(ResultSet resultSet, long maxCacheSize, FsPath storePath) { + super(resultSet, maxCacheSize, storePath); + this.resultSet = resultSet; + this.maxCacheSize = maxCacheSize; + this.storePath = storePath; + } + + @Override + public void addMetaData(MetaData metaData) throws IOException { + if (!moveToWriteRow) { + this.metaData = metaData; + if (this.schema == null) { + this.schema = TypeDescription.createStruct(); + } + TableMetaData tableMetaData = (TableMetaData) this.metaData; + for (Column column : tableMetaData.columns) { + schema.addField(column.getColumnName(), OrcUtils.dataTypeToOrcType(column.getDataType())); + } + moveToWriteRow = true; + if (writer == null) { + writer = + OrcFile.createWriter( + new Path(storePath.getPath()), + OrcFile.writerOptions(new Configuration()) + .setSchema(schema) + .compress(CompressionKind.ZLIB) + .version(OrcFile.Version.V_0_12)); + } + } + } + + @Override + public void addRecord(Record record) { + if (moveToWriteRow) { + TableRecord tableRecord = (TableRecord) record; + TableMetaData tableMetaData = (TableMetaData) metaData; + try { + Object[] row = tableRecord.row; + VectorizedRowBatch batch = schema.createRowBatch(); + int rowCount = batch.size++; + + for (int i = 0; i < row.length; i++) { + OrcUtils.setColumn( + rowCount, batch.cols[i], tableMetaData.columns[i].getDataType(), row[i]); + if (batch.size == batch.getMaxSize()) { + writer.addRowBatch(batch); + batch.reset(); + } + } + writer.addRowBatch(batch); + + } catch (IOException e) { + logger.warn("addMetaDataAndRecordString failed", e); + } + } + } + + @Override + public FsPath toFSPath() { + return storePath; + } + + @Override + public String toString() { + return storePath.getSchemaPath(); + } + + @Override + public void addMetaDataAndRecordString(String content) {} + + @Override + public void addRecordString(String content) {} + + @Override + public void close() throws IOException { + writer.close(); + } + + @Override + public void flush() throws IOException {} +} diff --git a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/resultset/ParquetResultSetReader.java b/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/resultset/ParquetResultSetReader.java new file mode 100644 index 0000000000..c09804294d --- /dev/null +++ b/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/resultset/ParquetResultSetReader.java @@ -0,0 +1,158 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.storage.resultset; + +import org.apache.linkis.common.io.FsPath; +import org.apache.linkis.common.io.MetaData; +import org.apache.linkis.common.io.Record; +import org.apache.linkis.common.io.resultset.ResultSet; +import org.apache.linkis.common.io.resultset.ResultSetReader; +import org.apache.linkis.storage.domain.Column; +import org.apache.linkis.storage.domain.DataType; +import org.apache.linkis.storage.resultset.table.TableMetaData; +import org.apache.linkis.storage.resultset.table.TableRecord; + +import org.apache.avro.Schema; +import org.apache.avro.generic.GenericRecord; +import org.apache.commons.io.IOUtils; +import org.apache.hadoop.fs.Path; +import org.apache.parquet.avro.AvroParquetReader; +import org.apache.parquet.hadoop.ParquetReader; + +import java.io.IOException; +import java.io.InputStream; +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class ParquetResultSetReader + extends ResultSetReader { + + private static final Logger logger = LoggerFactory.getLogger(ParquetResultSetReader.class); + + private FsPath fsPath; + + private final ResultSet resultSet; + + private final InputStream inputStream; + + private MetaData metaData; + + private Record row; + + private ParquetReader parquetReader; + + private GenericRecord record; + + public ParquetResultSetReader(ResultSet resultSet, InputStream inputStream, FsPath fsPath) + throws IOException { + super(resultSet, inputStream); + this.resultSet = resultSet; + this.inputStream = inputStream; + this.fsPath = fsPath; + this.parquetReader = + AvroParquetReader.builder(new Path(fsPath.getPath())).build(); + this.record = parquetReader.read(); + } + + @Override + public MetaData getMetaData() { + if (metaData == null) { + try { + List fields = record.getSchema().getFields(); + List columnList = + fields.stream() + .map( + field -> + new Column( + field.name(), + DataType.toDataType(field.schema().getType().getName()), + "")) + .collect(Collectors.toList()); + + metaData = new TableMetaData(columnList.toArray(new Column[0])); + } catch (Exception e) { + throw new RuntimeException("Failed to read parquet schema", e); + } + } + return metaData; + } + + @Override + public int skip(int recordNum) throws IOException { + if (recordNum < 0) return -1; + + for (int i = recordNum; i > 0; i--) { + try { + this.record = parquetReader.read(); + } catch (Throwable t) { + return recordNum - i; + } + } + return recordNum; + } + + @Override + public long getPosition() throws IOException { + throw new UnsupportedOperationException("Storeage Unsupported type: getPosition"); + } + + @Override + public long available() throws IOException { + throw new UnsupportedOperationException("Storeage Unsupported type: available"); + } + + @Override + public boolean hasNext() throws IOException { + if (metaData == null) getMetaData(); + if (record == null) return false; + ArrayList resultList = new ArrayList<>(); + TableMetaData tableMetaData = (TableMetaData) metaData; + int length = tableMetaData.getColumns().length; + for (int i = 0; i < length; i++) { + resultList.add(record.get(i)); + } + row = new TableRecord(resultList.toArray(new Object[0])); + if (row == null) return false; + return record != null; + } + + @Override + public Record getRecord() { + if (metaData == null) throw new RuntimeException("Must read metadata first(必须先读取metadata)"); + if (row == null) { + throw new RuntimeException( + "Can't get the value of the field, maybe the IO stream has been read or has been closed!(拿不到字段的值，也许IO流已读取完毕或已被关闭！)"); + } + try { + this.record = parquetReader.read(); + } catch (IOException e) { + throw new RuntimeException("Failed to read parquet record", e); + } + return row; + } + + @Override + public void close() throws IOException { + IOUtils.closeQuietly(inputStream); + parquetReader.close(); + } +} diff --git a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/resultset/ParquetResultSetWriter.java b/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/resultset/ParquetResultSetWriter.java new file mode 100644 index 0000000000..6fbac3c8cb --- /dev/null +++ b/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/resultset/ParquetResultSetWriter.java @@ -0,0 +1,135 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.storage.resultset; + +import org.apache.linkis.common.io.FsPath; +import org.apache.linkis.common.io.MetaData; +import org.apache.linkis.common.io.Record; +import org.apache.linkis.common.io.resultset.ResultSet; +import org.apache.linkis.common.io.resultset.ResultSetWriter; +import org.apache.linkis.storage.domain.Column; +import org.apache.linkis.storage.resultset.table.TableMetaData; +import org.apache.linkis.storage.resultset.table.TableRecord; + +import org.apache.avro.Schema; +import org.apache.avro.SchemaBuilder; +import org.apache.avro.generic.GenericData; +import org.apache.avro.generic.GenericRecord; +import org.apache.hadoop.fs.Path; +import org.apache.parquet.avro.AvroParquetWriter; +import org.apache.parquet.hadoop.ParquetFileWriter; +import org.apache.parquet.hadoop.ParquetWriter; +import org.apache.parquet.hadoop.metadata.CompressionCodecName; + +import java.io.IOException; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class ParquetResultSetWriter + extends ResultSetWriter { + private static final Logger logger = LoggerFactory.getLogger(ParquetResultSetWriter.class); + + private Schema schema; + + private ParquetWriter parquetWriter; + + private boolean moveToWriteRow = false; + + private MetaData metaData = null; + + private final FsPath storePath; + + private final long maxCacheSize; + + private final ResultSet resultSet; + + public ParquetResultSetWriter(ResultSet resultSet, long maxCacheSize, FsPath storePath) { + super(resultSet, maxCacheSize, storePath); + this.resultSet = resultSet; + this.maxCacheSize = maxCacheSize; + this.storePath = storePath; + } + + @Override + public void addMetaData(MetaData metaData) throws IOException { + if (!moveToWriteRow) { + this.metaData = metaData; + SchemaBuilder.FieldAssembler fieldAssembler = SchemaBuilder.record("linkis").fields(); + TableMetaData tableMetaData = (TableMetaData) this.metaData; + for (Column column : tableMetaData.columns) { + fieldAssembler + .name(column.getColumnName().replaceAll("\\.", "_").replaceAll("[^a-zA-Z0-9_]", "")) + .doc(column.getComment()) + .type(column.getDataType().getTypeName().toLowerCase()) + .noDefault(); + } + schema = fieldAssembler.endRecord(); + moveToWriteRow = true; + if (parquetWriter == null) { + parquetWriter = + AvroParquetWriter.builder(new Path(storePath.getPath())) + .withSchema(schema) + .withCompressionCodec(CompressionCodecName.SNAPPY) + .withWriteMode(ParquetFileWriter.Mode.OVERWRITE) + .build(); + } + } + } + + @Override + public void addRecord(Record record) { + if (moveToWriteRow) { + TableRecord tableRecord = (TableRecord) record; + try { + Object[] row = tableRecord.row; + GenericRecord genericRecord = new GenericData.Record(schema); + for (int i = 0; i < row.length; i++) { + genericRecord.put(schema.getFields().get(i).name(), row[i]); + } + parquetWriter.write(genericRecord); + } catch (IOException e) { + logger.warn("addMetaDataAndRecordString failed", e); + } + } + } + + @Override + public FsPath toFSPath() { + return storePath; + } + + @Override + public String toString() { + return storePath.getSchemaPath(); + } + + @Override + public void addMetaDataAndRecordString(String content) {} + + @Override + public void addRecordString(String content) {} + + @Override + public void close() throws IOException { + parquetWriter.close(); + } + + @Override + public void flush() throws IOException {} +} diff --git a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/resultset/ResultSetReaderFactory.java b/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/resultset/ResultSetReaderFactory.java index 3047b715a0..bf46f49f9b 100644 --- a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/resultset/ResultSetReaderFactory.java +++ b/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/resultset/ResultSetReaderFactory.java @@ -24,7 +24,11 @@ import org.apache.linkis.common.io.resultset.ResultSet; import org.apache.linkis.common.io.resultset.ResultSetReader; import org.apache.linkis.storage.FSFactory; +import org.apache.linkis.storage.conf.LinkisStorageConf; +import org.apache.linkis.storage.domain.Dolphin; import org.apache.linkis.storage.errorcode.LinkisStorageErrorCodeSummary; +import org.apache.linkis.storage.exception.StorageErrorCode; +import org.apache.linkis.storage.exception.StorageReadException; import org.apache.linkis.storage.exception.StorageWarnException; import org.apache.linkis.storage.resultset.table.TableMetaData; import org.apache.linkis.storage.resultset.table.TableRecord; @@ -40,8 +44,30 @@ public class ResultSetReaderFactory { private static final Logger logger = LoggerFactory.getLogger(ResultSetReaderFactory.class); public static ResultSetReader getResultSetReader( - ResultSet resultSet, InputStream inputStream) { - return new StorageResultSetReader<>(resultSet, inputStream); + ResultSet resultSet, InputStream inputStream, FsPath fsPath) { + ResultSetReader resultSetReader = null; + if (fsPath.getPath().endsWith(Dolphin.DOLPHIN_FILE_SUFFIX)) { + resultSetReader = new StorageResultSetReader<>(resultSet, inputStream); + } else if (fsPath.getPath().endsWith(LinkisStorageConf.PARQUET_FILE_SUFFIX)) { + try { + resultSetReader = new ParquetResultSetReader<>(resultSet, inputStream, fsPath); + } catch (IOException e) { + throw new StorageReadException( + StorageErrorCode.READ_PARQUET_FAILED.getCode(), + StorageErrorCode.READ_PARQUET_FAILED.getMessage(), + e); + } + } else if (fsPath.getPath().endsWith(LinkisStorageConf.ORC_FILE_SUFFIX)) { + try { + resultSetReader = new OrcResultSetReader<>(resultSet, inputStream, fsPath); + } catch (IOException e) { + throw new StorageReadException( + StorageErrorCode.READ_ORC_FAILED.getCode(), + StorageErrorCode.READ_ORC_FAILED.getMessage(), + e); + } + } + return resultSetReader; } public static ResultSetReader getResultSetReader( @@ -49,7 +75,7 @@ public static ResultSetReader getResultSe return new StorageResultSetReader<>(resultSet, value); } - public static ResultSetReader getResultSetReader(String res) { + public static ResultSetReader getResultSetReader(String res) throws IOException { ResultSetFactory rsFactory = ResultSetFactory.getInstance(); if (rsFactory.isResultSet(res)) { ResultSet resultSet = rsFactory.getResultSet(res); @@ -58,21 +84,12 @@ public static ResultSetReader getResultSetReader(String res) { FsPath resPath = new FsPath(res); ResultSet resultSet = rsFactory.getResultSetByPath(resPath); - try { - FSFactory.getFs(resPath).init(null); - } catch (IOException e) { - logger.warn("ResultSetReaderFactory fs init failed", e); - } - ResultSetReader reader = null; - try { - reader = - ResultSetReaderFactory.getResultSetReader( - resultSet, FSFactory.getFs(resPath).read(resPath)); - } catch (IOException e) { - logger.warn("ResultSetReaderFactory fs read failed", e); - } + Fs fs = FSFactory.getFs(resPath); + fs.init(null); + ResultSetReader reader = + ResultSetReaderFactory.getResultSetReader(resultSet, fs.read(resPath), resPath); if (reader instanceof StorageResultSetReader) { - ((StorageResultSetReader) reader).setFs(FSFactory.getFs(resPath)); + ((StorageResultSetReader) reader).setFs(fs); } return (StorageResultSetReader) reader; } @@ -105,7 +122,7 @@ public static ResultSetReader getTableResultReader(String res) { InputStream read = fs.read(resPath); return ResultSetReaderFactory.getResultSetReader( - (TableResultSet) resultSet, read); + (TableResultSet) resultSet, read, resPath); } catch (IOException e) { throw new StorageWarnException( LinkisStorageErrorCodeSummary.TABLE_ARE_NOT_SUPPORTED.getErrorCode(), diff --git a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/resultset/ResultSetWriterFactory.java b/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/resultset/ResultSetWriterFactory.java index 1abeaf0937..9cb1999367 100644 --- a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/resultset/ResultSetWriterFactory.java +++ b/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/resultset/ResultSetWriterFactory.java @@ -22,6 +22,9 @@ import org.apache.linkis.common.io.Record; import org.apache.linkis.common.io.resultset.ResultSet; import org.apache.linkis.common.io.resultset.ResultSetReader; +import org.apache.linkis.common.io.resultset.ResultSetWriter; +import org.apache.linkis.storage.conf.LinkisStorageConf; +import org.apache.linkis.storage.resultset.table.TableResultSet; import java.io.IOException; import java.util.ArrayList; @@ -33,46 +36,57 @@ public class ResultSetWriterFactory { private static final Logger logger = LoggerFactory.getLogger(ResultSetWriterFactory.class); - public static - org.apache.linkis.common.io.resultset.ResultSetWriter getResultSetWriter( - ResultSet resultSet, long maxCacheSize, FsPath storePath) { - return new StorageResultSetWriter<>(resultSet, maxCacheSize, storePath); + public static ResultSetWriter getResultSetWriter( + ResultSet resultSet, long maxCacheSize, FsPath storePath) { + String engineResultType = LinkisStorageConf.ENGINE_RESULT_TYPE; + ResultSetWriter writer = null; + if (engineResultType.equals(LinkisStorageConf.PARQUET) && resultSet instanceof TableResultSet) { + writer = new ParquetResultSetWriter<>(resultSet, maxCacheSize, storePath); + } else if (engineResultType.equals(LinkisStorageConf.ORC) + && resultSet instanceof TableResultSet) { + writer = new OrcResultSetWriter<>(resultSet, maxCacheSize, storePath); + } else { + writer = new StorageResultSetWriter<>(resultSet, maxCacheSize, storePath); + } + return writer; } - public static - org.apache.linkis.common.io.resultset.ResultSetWriter getResultSetWriter( - ResultSet resultSet, long maxCacheSize, FsPath storePath, String proxyUser) { - StorageResultSetWriter writer = - new StorageResultSetWriter<>(resultSet, maxCacheSize, storePath); - writer.setProxyUser(proxyUser); + public static ResultSetWriter getResultSetWriter( + ResultSet resultSet, long maxCacheSize, FsPath storePath, String proxyUser) { + String engineResultType = LinkisStorageConf.ENGINE_RESULT_TYPE; + ResultSetWriter writer = null; + if (engineResultType.equals(LinkisStorageConf.PARQUET) && resultSet instanceof TableResultSet) { + writer = new ParquetResultSetWriter<>(resultSet, maxCacheSize, storePath); + } else if (engineResultType.equals(LinkisStorageConf.ORC) + && resultSet instanceof TableResultSet) { + writer = new OrcResultSetWriter<>(resultSet, maxCacheSize, storePath); + } else { + writer = new StorageResultSetWriter<>(resultSet, maxCacheSize, storePath); + StorageResultSetWriter storageResultSetWriter = (StorageResultSetWriter) writer; + storageResultSetWriter.setProxyUser(proxyUser); + } return writer; } public static Record[] getRecordByWriter( - org.apache.linkis.common.io.resultset.ResultSetWriter - writer, - long limit) { + ResultSetWriter writer, long limit) throws IOException { String res = writer.toString(); return getRecordByRes(res, limit); } - public static Record[] getRecordByRes(String res, long limit) { + public static Record[] getRecordByRes(String res, long limit) throws IOException { ResultSetReader reader = ResultSetReaderFactory.getResultSetReader(res); int count = 0; List records = new ArrayList<>(); - try { - reader.getMetaData(); - while (reader.hasNext() && count < limit) { - records.add(reader.getRecord()); - count++; - } - } catch (IOException e) { - logger.warn("ResultSetWriter getRecordByRes failed", e); + reader.getMetaData(); + while (reader.hasNext() && count < limit) { + records.add(reader.getRecord()); + count++; } return records.toArray(new Record[0]); } - public static Record getLastRecordByRes(String res) { + public static Record getLastRecordByRes(String res) throws IOException { ResultSetReader reader = ResultSetReaderFactory.getResultSetReader(res); Record record = null; try { diff --git a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/resultset/StorageResultSet.java b/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/resultset/StorageResultSet.java index c83661de2e..c708f5faf6 100644 --- a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/resultset/StorageResultSet.java +++ b/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/resultset/StorageResultSet.java @@ -21,7 +21,9 @@ import org.apache.linkis.common.io.MetaData; import org.apache.linkis.common.io.Record; import org.apache.linkis.common.io.resultset.ResultSet; +import org.apache.linkis.storage.conf.LinkisStorageConf; import org.apache.linkis.storage.domain.Dolphin; +import org.apache.linkis.storage.resultset.table.TableResultSet; import org.apache.linkis.storage.utils.StorageConfiguration; import org.slf4j.Logger; @@ -49,10 +51,17 @@ public String charset() { @Override public FsPath getResultSetPath(FsPath parentDir, String fileName) { + String engineResultType = LinkisStorageConf.ENGINE_RESULT_TYPE; + String fileSuffix = Dolphin.DOLPHIN_FILE_SUFFIX; + if (engineResultType.equals(LinkisStorageConf.PARQUET) && this instanceof TableResultSet) { + fileSuffix = LinkisStorageConf.PARQUET_FILE_SUFFIX; + } else if (engineResultType.equals(LinkisStorageConf.ORC) && this instanceof TableResultSet) { + fileSuffix = LinkisStorageConf.ORC_FILE_SUFFIX; + } final String path = parentDir.getPath().endsWith("/") - ? parentDir.getUriString() + fileName + Dolphin.DOLPHIN_FILE_SUFFIX - : parentDir.getUriString() + "/" + fileName + Dolphin.DOLPHIN_FILE_SUFFIX; + ? parentDir.getUriString() + fileName + fileSuffix + : parentDir.getUriString() + "/" + fileName + fileSuffix; logger.info("Get result set path: {}", path); return new FsPath(path); } diff --git a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/resultset/StorageResultSetWriter.java b/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/resultset/StorageResultSetWriter.java index 5109ed44df..230c68301c 100644 --- a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/resultset/StorageResultSetWriter.java +++ b/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/resultset/StorageResultSetWriter.java @@ -17,14 +17,19 @@ package org.apache.linkis.storage.resultset; -import org.apache.linkis.common.io.*; -import org.apache.linkis.common.io.resultset.*; +import org.apache.linkis.common.io.Fs; +import org.apache.linkis.common.io.FsPath; +import org.apache.linkis.common.io.MetaData; +import org.apache.linkis.common.io.Record; +import org.apache.linkis.common.io.resultset.ResultSerializer; +import org.apache.linkis.common.io.resultset.ResultSet; import org.apache.linkis.common.io.resultset.ResultSetWriter; -import org.apache.linkis.common.utils.*; -import org.apache.linkis.storage.*; -import org.apache.linkis.storage.conf.*; -import org.apache.linkis.storage.domain.*; -import org.apache.linkis.storage.utils.*; +import org.apache.linkis.storage.FSFactory; +import org.apache.linkis.storage.conf.LinkisStorageConf; +import org.apache.linkis.storage.domain.Dolphin; +import org.apache.linkis.storage.exception.StorageErrorException; +import org.apache.linkis.storage.utils.FileSystemUtils; +import org.apache.linkis.storage.utils.StorageUtils; import org.apache.commons.io.IOUtils; import org.apache.hadoop.hdfs.client.HdfsDataOutputStream; @@ -37,6 +42,8 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import static org.apache.linkis.storage.exception.StorageErrorCode.FS_ERROR; + public class StorageResultSetWriter extends ResultSetWriter { private static final Logger logger = LoggerFactory.getLogger(StorageResultSetWriter.class); @@ -98,8 +105,9 @@ public void createNewFile() { fs.init(null); FileSystemUtils.createNewFile(storePath, proxyUser, true); outputStream = fs.write(storePath, true); - } catch (IOException e) { - logger.warn("StorageResultSetWriter createNewFile failed", e); + } catch (Exception e) { + throw new StorageErrorException( + FS_ERROR.getCode(), "StorageResultSetWriter createNewFile failed", e); } logger.info("Succeed to create a new file:{}", storePath); fileCreated = true; diff --git a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/source/FileSource.java b/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/source/FileSource.java index 0ed650186d..f7a5d3e512 100644 --- a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/source/FileSource.java +++ b/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/source/FileSource.java @@ -73,7 +73,9 @@ public interface FileSource extends Closeable { (path, suffix) -> path.endsWith("." + suffix); static boolean isResultSet(String path) { - return suffixPredicate.apply(path, fileType[0]); + return suffixPredicate.apply(path, LinkisStorageConf.DOLPHIN) + || suffixPredicate.apply(path, LinkisStorageConf.PARQUET) + || suffixPredicate.apply(path, LinkisStorageConf.ORC); } static boolean isResultSet(FsPath fsPath) { @@ -137,7 +139,8 @@ static FileSource create(FsPath fsPath, InputStream is) { static FileSplit createResultSetFileSplit(FsPath fsPath, InputStream is) { logger.info("try create result set file split with path:{}", fsPath.getPath()); ResultSet resultset = ResultSetFactory.getInstance().getResultSetByPath(fsPath); - ResultSetReader resultsetReader = ResultSetReaderFactory.getResultSetReader(resultset, is); + ResultSetReader resultsetReader = + ResultSetReaderFactory.getResultSetReader(resultset, is, fsPath); return new FileSplit(resultsetReader, resultset.resultSetType()); } @@ -145,7 +148,8 @@ static FileSplit createResultSetFileSplit(FsPath fsPath, Fs fs) { ResultSet resultset = ResultSetFactory.getInstance().getResultSetByPath(fsPath, fs); ResultSetReader resultsetReader = null; try { - resultsetReader = ResultSetReaderFactory.getResultSetReader(resultset, fs.read(fsPath)); + resultsetReader = + ResultSetReaderFactory.getResultSetReader(resultset, fs.read(fsPath), fsPath); } catch (IOException e) { logger.warn("FileSource createResultSetFileSplit failed", e); } diff --git a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/source/ResultsetFileSource.java b/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/source/ResultsetFileSource.java index fb064a8f4f..54fd64daad 100644 --- a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/source/ResultsetFileSource.java +++ b/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/source/ResultsetFileSource.java @@ -47,7 +47,7 @@ record -> { if (emptyValue.equals(Dolphin.LINKIS_NULL)) { return ""; } else { - return nullValue; + return emptyValue; } } else if (r instanceof Double) { return StorageUtils.doubleToString((Double) r); diff --git a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/utils/FileSystemUtils.java b/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/utils/FileSystemUtils.java index 2809c83eec..4c50479637 100644 --- a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/utils/FileSystemUtils.java +++ b/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/utils/FileSystemUtils.java @@ -61,16 +61,12 @@ public static void createNewFile(FsPath filePath, boolean createParentWhenNotExi createNewFile(filePath, StorageUtils.getJvmUser(), createParentWhenNotExists); } - public static void createNewFile( - FsPath filePath, String user, boolean createParentWhenNotExists) { + public static void createNewFile(FsPath filePath, String user, boolean createParentWhenNotExists) + throws Exception { FileSystem fileSystem = (FileSystem) FSFactory.getFsByProxyUser(filePath, user); try { fileSystem.init(null); createNewFileWithFileSystem(fileSystem, filePath, user, createParentWhenNotExists); - } catch (IOException e) { - logger.warn("FileSystemUtils createNewFile failed", e); - } catch (Exception e) { - logger.warn("FileSystemUtils createNewFile failed", e); } finally { IOUtils.closeQuietly(fileSystem); } diff --git a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/utils/OrcUtils.java b/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/utils/OrcUtils.java new file mode 100644 index 0000000000..72c2f8c355 --- /dev/null +++ b/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/utils/OrcUtils.java @@ -0,0 +1,152 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.storage.utils; + +import org.apache.linkis.storage.domain.DataType; + +import org.apache.orc.TypeDescription; +import org.apache.orc.storage.common.type.HiveDecimal; +import org.apache.orc.storage.ql.exec.vector.*; + +import java.math.BigDecimal; +import java.sql.Timestamp; +import java.text.ParseException; +import java.text.SimpleDateFormat; + +/** + * Inspired by: + * https://github.com/apache/flink/blob/master/flink-formats/flink-orc/src/main/java/org/apache/flink/orc/OrcSplitReaderUtil.java + */ +public class OrcUtils { + + public static TypeDescription dataTypeToOrcType(DataType type) { + switch (type) { + case CharType: + return TypeDescription.createChar().withMaxLength(1024); + case StringType: + return TypeDescription.createString(); + case LongType: + return TypeDescription.createLong(); + case VarcharType: + return TypeDescription.createVarchar().withMaxLength(1024); + case BooleanType: + return TypeDescription.createBoolean(); + case BinaryType: + return TypeDescription.createBinary(); + case DecimalType: + return TypeDescription.createDecimal().withScale(10).withPrecision(38); + case TinyIntType: + return TypeDescription.createByte(); + case ShortIntType: + return TypeDescription.createShort(); + case IntType: + return TypeDescription.createInt(); + case BigIntType: + return TypeDescription.createLong(); + case FloatType: + return TypeDescription.createFloat(); + case DoubleType: + return TypeDescription.createDouble(); + case DateType: + return TypeDescription.createDate(); + case TimestampType: + return TypeDescription.createTimestamp(); + case ArrayType: + return TypeDescription.createList(dataTypeToOrcType(DataType.VarcharType)); + case MapType: + return TypeDescription.createMap( + dataTypeToOrcType(DataType.VarcharType), dataTypeToOrcType(DataType.VarcharType)); + default: + throw new UnsupportedOperationException("Unsupported type: " + type); + } + } + + public static void setColumn(int columnId, ColumnVector column, DataType type, Object value) { + switch (type) { + case CharType: + case VarcharType: + case BinaryType: + case StringType: + { + BytesColumnVector vector = (BytesColumnVector) column; + vector.setVal(columnId, String.valueOf(value).getBytes()); + break; + } + case BooleanType: + { + LongColumnVector vector = (LongColumnVector) column; + vector.vector[columnId] = Boolean.valueOf(value.toString()) ? 1 : 0; + break; + } + case DecimalType: + { + DecimalColumnVector vector = (DecimalColumnVector) column; + vector.set(columnId, HiveDecimal.create(new BigDecimal(value.toString()))); + break; + } + case TinyIntType: + { + LongColumnVector vector = (LongColumnVector) column; + vector.vector[columnId] = (byte) value; + break; + } + case DateType: + case IntType: + { + LongColumnVector vector = (LongColumnVector) column; + vector.vector[columnId] = Integer.valueOf(value.toString()); + break; + } + case BigIntType: + { + LongColumnVector vector = (LongColumnVector) column; + vector.vector[columnId] = Long.valueOf(value.toString()); + break; + } + case FloatType: + { + DoubleColumnVector vector = (DoubleColumnVector) column; + vector.vector[columnId] = Float.valueOf(value.toString()); + break; + } + case DoubleType: + { + DoubleColumnVector vector = (DoubleColumnVector) column; + vector.vector[columnId] = Double.valueOf(value.toString()); + break; + } + case TimestampType: + { + TimestampColumnVector vector = (TimestampColumnVector) column; + try { + vector.set( + columnId, + new Timestamp( + new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSSZ") + .parse(value.toString()) + .getTime())); + } catch (ParseException e) { + vector.set(columnId, new Timestamp(System.currentTimeMillis())); + } + break; + } + default: + throw new UnsupportedOperationException("Unsupported type: " + type); + } + } +} diff --git a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/utils/StorageHelper.java b/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/utils/StorageHelper.java index e1dee151ca..491c3d7af4 100644 --- a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/utils/StorageHelper.java +++ b/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/utils/StorageHelper.java @@ -80,7 +80,8 @@ public static void getTableResLines(String[] args) { resultSetFactory.getResultSetByType(ResultSetFactory.TABLE_TYPE); Fs fs = FSFactory.getFs(resPath); fs.init(null); - resultSetReader = ResultSetReaderFactory.getResultSetReader(resultSet, fs.read(resPath)); + resultSetReader = + ResultSetReaderFactory.getResultSetReader(resultSet, fs.read(resPath), resPath); TableMetaData metaData = (TableMetaData) resultSetReader.getMetaData(); Arrays.stream(metaData.getColumns()).forEach(column -> logger.info(column.toString())); int num = 0; @@ -116,7 +117,7 @@ public static void getTableRes(String[] args) { fs.init(null); ResultSetReader reader = - ResultSetReaderFactory.getResultSetReader(resultSet, fs.read(resPath)); + ResultSetReaderFactory.getResultSetReader(resultSet, fs.read(resPath), resPath); MetaData rmetaData = reader.getMetaData(); Arrays.stream(((TableMetaData) rmetaData).getColumns()) .forEach(column -> logger.info(column.toString())); diff --git a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/utils/StorageUtils.java b/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/utils/StorageUtils.java index 692ce619b2..07bc0510bc 100644 --- a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/utils/StorageUtils.java +++ b/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/utils/StorageUtils.java @@ -272,4 +272,8 @@ public static byte[] mergeByteArrays(byte[] arr1, byte[] arr2) { System.arraycopy(arr2, 0, mergedArray, arr1.length, arr2.length); return mergedArray; } + + public static boolean isHDFSPath(FsPath fsPath) { + return HDFS.equalsIgnoreCase(fsPath.getFsType()); + } } diff --git a/linkis-computation-governance/linkis-computation-governance-common/src/main/java/org/apache/linkis/governance/common/constant/CodeConstants.java b/linkis-computation-governance/linkis-computation-governance-common/src/main/java/org/apache/linkis/governance/common/constant/CodeConstants.java new file mode 100644 index 0000000000..4c914bc3f4 --- /dev/null +++ b/linkis-computation-governance/linkis-computation-governance-common/src/main/java/org/apache/linkis/governance/common/constant/CodeConstants.java @@ -0,0 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.governance.common.constant; + +public class CodeConstants { + // will auto append at end of scala code; make sure the last line is not a comment + public static String SCALA_CODE_AUTO_APPEND_CODE = "val linkisVar=123"; +} diff --git a/linkis-computation-governance/linkis-computation-governance-common/src/main/scala/org/apache/linkis/governance/common/paser/CodeParser.scala b/linkis-computation-governance/linkis-computation-governance-common/src/main/scala/org/apache/linkis/governance/common/paser/CodeParser.scala index 64ece62fd7..d5669ad428 100644 --- a/linkis-computation-governance/linkis-computation-governance-common/src/main/scala/org/apache/linkis/governance/common/paser/CodeParser.scala +++ b/linkis-computation-governance/linkis-computation-governance-common/src/main/scala/org/apache/linkis/governance/common/paser/CodeParser.scala @@ -19,6 +19,7 @@ package org.apache.linkis.governance.common.paser import org.apache.linkis.common.utils.{CodeAndRunTypeUtils, Logging, Utils} import org.apache.linkis.governance.common.conf.GovernanceCommonConf +import org.apache.linkis.governance.common.constant.CodeConstants import org.apache.linkis.governance.common.paser.CodeType.CodeType import org.apache.commons.lang3.StringUtils @@ -116,7 +117,7 @@ class ScalaCodeParser extends SingleCodeParser with Logging { if (statementBuffer.nonEmpty) codeBuffer.append(statementBuffer.mkString("\n")) // Make sure the last line is not a comment codeBuffer.append("\n") - codeBuffer.append("val linkisVar=123") + codeBuffer.append(CodeConstants.SCALA_CODE_AUTO_APPEND_CODE) codeBuffer.toArray } diff --git a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-core/src/main/scala/org/apache/linkis/ecm/core/launch/ProcessEngineConnLaunch.scala b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-core/src/main/scala/org/apache/linkis/ecm/core/launch/ProcessEngineConnLaunch.scala index 672c0e8acb..5b23d01fc3 100644 --- a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-core/src/main/scala/org/apache/linkis/ecm/core/launch/ProcessEngineConnLaunch.scala +++ b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-core/src/main/scala/org/apache/linkis/ecm/core/launch/ProcessEngineConnLaunch.scala @@ -176,8 +176,15 @@ trait ProcessEngineConnLaunch extends EngineConnLaunch with Logging { .findAvailPortByRange(GovernanceCommonConf.ENGINE_CONN_PORT_RANGE.getValue) .toString - var springConf = Map("server.port" -> engineConnPort, "spring.profiles.active" -> "engineconn") - + var springConf = + Map[String, String]("server.port" -> engineConnPort, "spring.profiles.active" -> "engineconn") + val properties = + PortUtils.readFromProperties(Configuration.getLinkisHome + "/conf/version.properties") + if (StringUtils.isNotBlank(properties.getProperty("version"))) { + springConf += ("eureka.instance.metadata-map.linkis.app.version" -> properties.getProperty( + "version" + )) + } request.creationDesc.properties.asScala.filter(_._1.startsWith("spring.")).foreach { case (k, v) => springConf = springConf + (k -> v) diff --git a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-core/src/main/scala/org/apache/linkis/ecm/core/utils/PortUtils.scala b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-core/src/main/scala/org/apache/linkis/ecm/core/utils/PortUtils.scala index b3715a8910..21dd6cd706 100644 --- a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-core/src/main/scala/org/apache/linkis/ecm/core/utils/PortUtils.scala +++ b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-core/src/main/scala/org/apache/linkis/ecm/core/utils/PortUtils.scala @@ -17,15 +17,16 @@ package org.apache.linkis.ecm.core.utils -import org.apache.linkis.common.utils.Utils +import org.apache.linkis.common.utils.{Logging, Utils} import org.apache.commons.io.IOUtils import org.apache.commons.lang3.StringUtils -import java.io.IOException +import java.io.{BufferedReader, FileReader, IOException} import java.net.ServerSocket +import java.util.Properties -object PortUtils { +object PortUtils extends Logging { /** * portRange: '-' is the separator @@ -62,4 +63,23 @@ object PortUtils { Utils.tryFinally(socket.getLocalPort)(IOUtils.closeQuietly(socket)) } + def readFromProperties(propertiesFile: String): Properties = { + val properties: Properties = new Properties + var reader: BufferedReader = null; + try { + reader = new BufferedReader(new FileReader(propertiesFile)) + properties.load(reader) + } catch { + case e: Exception => + logger.warn(s"loading vsersion faild with path $propertiesFile error:$e") + } finally { + try if (reader != null) reader.close + catch { + case e: Exception => + logger.warn(s"try to close buffered reader with error:${e.getMessage}") + } + } + properties + } + } diff --git a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/pom.xml b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/pom.xml index 41022d30da..99458c8afc 100644 --- a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/pom.xml +++ b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/pom.xml @@ -51,6 +51,12 @@ linkis-rpc ${project.version} provided + + + com.google.guava + guava + + diff --git a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/java/org/apache/linkis/ecm/errorcode/EngineconnServerErrorCodeSummary.java b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/java/org/apache/linkis/ecm/errorcode/EngineconnServerErrorCodeSummary.java index ca4412824d..0c745ef64d 100644 --- a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/java/org/apache/linkis/ecm/errorcode/EngineconnServerErrorCodeSummary.java +++ b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/java/org/apache/linkis/ecm/errorcode/EngineconnServerErrorCodeSummary.java @@ -33,7 +33,15 @@ public enum EngineconnServerErrorCodeSummary implements LinkisErrorCode { 11110, "the parameters of engineConnInstance and ticketId are both not exists.(engineConnInstance 和ticketId 的参数都不存在.)"), LOG_IS_NOT_EXISTS(11110, "Log directory {0} does not exists.(日志目录 {0} 不存在.)"), - FAILED_TO_DOWNLOAD(911115, "failed to downLoad(下载失败)"); + FAILED_TO_DOWNLOAD(911115, "failed to downLoad(下载失败)"), + FILE_IS_OVERSIZE(911116, "Download file has exceeded 100MB(下载文件已超过100M)"), + PARAMETER_NOT_NULL(911117, "Parameter {0} cannot be empty (参数 {0} 不能为空)"), + LOGTYPE_ERROR( + 911118, + "logType only supports stdout, stderr, gc, yarnApp(logType仅支持stdout,stderr,gc,yarnApp)"), + NOT_PERMISSION( + 911119, "You {0} have no permission to download Log in ECM {1}(用户 {0} 无权限下载 ECM {1} 日志)"), + ; /** (errorCode)错误码 */ private final int errorCode; diff --git a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/java/org/apache/linkis/ecm/restful/ECMRestfulApi.java b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/java/org/apache/linkis/ecm/restful/ECMRestfulApi.java new file mode 100644 index 0000000000..d4fc0a49e1 --- /dev/null +++ b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/java/org/apache/linkis/ecm/restful/ECMRestfulApi.java @@ -0,0 +1,173 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.ecm.restful; + +import org.apache.linkis.common.conf.Configuration; +import org.apache.linkis.server.Message; +import org.apache.linkis.server.utils.ModuleUserUtils; + +import org.apache.commons.io.IOUtils; +import org.apache.commons.lang3.StringUtils; +import org.apache.http.Consts; + +import org.springframework.web.bind.annotation.*; + +import javax.servlet.ServletOutputStream; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + +import java.io.*; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.nio.file.attribute.FileOwnerAttributeView; +import java.nio.file.attribute.UserPrincipal; +import java.text.MessageFormat; + +import com.github.xiaoymin.knife4j.annotations.ApiOperationSupport; +import io.swagger.annotations.Api; +import io.swagger.annotations.ApiImplicitParam; +import io.swagger.annotations.ApiImplicitParams; +import io.swagger.annotations.ApiOperation; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import static org.apache.linkis.ecm.errorcode.EngineconnServerErrorCodeSummary.*; + +@Api(tags = "ECM") +@RequestMapping(path = "/engineconnManager") +@RestController +public class ECMRestfulApi { + + private final Logger logger = LoggerFactory.getLogger(ECMRestfulApi.class); + + /** + * * Reason for using the get method: Added gateway forwarding rules, which only support get + * requests + * + * @param req + * @param response + * @param emInstance + * @param instance + * @param logDirSuffix + * @param logType + * @throws IOException + */ + @ApiOperation( + value = "downloadEngineLog", + notes = "download engine log", + response = Message.class) + @ApiImplicitParams({ + @ApiImplicitParam( + name = "emInstance", + required = true, + dataType = "String", + example = "xxx0002:9102"), + @ApiImplicitParam( + name = "instance", + required = true, + dataType = "String", + example = "xxx0002:35873"), + @ApiImplicitParam(name = "logDirSuffix", required = true, dataType = "String"), + @ApiImplicitParam(name = "logType", required = true, dataType = "String") + }) + @ApiOperationSupport(ignoreParameters = {"json"}) + @RequestMapping(path = "/downloadEngineLog", method = RequestMethod.GET) + public Message downloadEngineLog( + HttpServletRequest req, + HttpServletResponse response, + @RequestParam(value = "emInstance") String emInstance, + @RequestParam(value = "instance") String instance, + @RequestParam(value = "logDirSuffix") String logDirSuffix, + @RequestParam(value = "logType") String logType) + throws IOException { + String userName = ModuleUserUtils.getOperationUser(req, "downloadEngineLog"); + if (StringUtils.isBlank(instance)) { + return Message.error(MessageFormat.format(PARAMETER_NOT_NULL.getErrorDesc(), "instance")); + } + if (StringUtils.isBlank(logDirSuffix)) { + return Message.error(MessageFormat.format(PARAMETER_NOT_NULL.getErrorDesc(), "logDirSuffix")); + } + if (StringUtils.isBlank(logType)) { + return Message.error(MessageFormat.format(PARAMETER_NOT_NULL.getErrorDesc(), "logType")); + } else if (!logType.equals("stdout") + && !logType.equals("stderr") + && !logType.equals("gc") + && !logType.equals("yarnApp")) { + return Message.error(LOGTYPE_ERROR.getErrorDesc()); + } + // 获取文件的权限归属者 + FileOwnerAttributeView ownerView = + Files.getFileAttributeView( + Paths.get(logDirSuffix + "/" + logType), FileOwnerAttributeView.class); + UserPrincipal owner = ownerView.getOwner(); + if (!owner.getName().equals(userName) + && Configuration.isNotAdmin(userName) + && Configuration.isNotJobHistoryAdmin(userName)) { + return Message.error( + MessageFormat.format(NOT_PERMISSION.getErrorDesc(), userName, emInstance)); + } + File inputFile = new File(logDirSuffix, logType); + if (!inputFile.exists()) { + return Message.error(MessageFormat.format(LOG_IS_NOT_EXISTS.getErrorDesc(), logDirSuffix)); + } else { + long fileSizeInBytes = inputFile.length(); + long fileSizeInMegabytes = fileSizeInBytes / (1024 * 1024); + if (fileSizeInMegabytes > 100) { + return Message.error(MessageFormat.format(FILE_IS_OVERSIZE.getErrorDesc(), logDirSuffix)); + } + ServletOutputStream outputStream = null; + FileInputStream inputStream = null; + BufferedInputStream fis = null; + PrintWriter writer = null; + try { + inputStream = new FileInputStream(inputFile); + fis = new BufferedInputStream(inputStream); + byte[] buffer = new byte[1024]; + int bytesRead = 0; + response.setCharacterEncoding(Consts.UTF_8.toString()); + java.nio.file.Path source = Paths.get(inputFile.getPath()); + response.addHeader("Content-Type", Files.probeContentType(source)); + // filename eg:xxx002_11529_stdout.txt + response.addHeader( + "Content-Disposition", + "attachment;filename=" + instance.replace(":", "_") + "_" + logType + ".txt"); + response.addHeader("Content-Length", fileSizeInBytes + ""); + outputStream = response.getOutputStream(); + while ((bytesRead = fis.read(buffer, 0, 1024)) != -1) { + outputStream.write(buffer, 0, bytesRead); + } + } catch (IOException e) { + logger.warn("Download EngineLog Failed Msg :", e); + response.reset(); + response.setCharacterEncoding(Consts.UTF_8.toString()); + response.setContentType("text/plain; charset=utf-8"); + writer = response.getWriter(); + writer.append("error(错误):" + e.getMessage()); + writer.flush(); + } finally { + if (outputStream != null) { + outputStream.flush(); + } + IOUtils.closeQuietly(outputStream); + IOUtils.closeQuietly(fis); + IOUtils.closeQuietly(inputStream); + } + return Message.ok(); + } + } +} diff --git a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/java/org/apache/linkis/ecm/server/service/impl/DefaultEngineConnKillService.java b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/java/org/apache/linkis/ecm/server/service/impl/DefaultEngineConnKillService.java index a6a932a578..5a0ade21dc 100644 --- a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/java/org/apache/linkis/ecm/server/service/impl/DefaultEngineConnKillService.java +++ b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/java/org/apache/linkis/ecm/server/service/impl/DefaultEngineConnKillService.java @@ -189,6 +189,7 @@ private String getYarnAppRegexByEngineType(String engineType) { case "sqoop": regex = EngineConnConf.SQOOP_ENGINE_CONN_YARN_APP_ID_PARSE_REGEX().getValue(); break; + case "flink": case "hive": regex = EngineConnConf.HIVE_ENGINE_CONN_YARN_APP_ID_PARSE_REGEX().getValue(); break; diff --git a/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/bml/BmlEnginePreExecuteHook.scala b/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/bml/BmlEnginePreExecuteHook.scala index 3959eb942b..cb04f364fb 100644 --- a/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/bml/BmlEnginePreExecuteHook.scala +++ b/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/bml/BmlEnginePreExecuteHook.scala @@ -62,7 +62,6 @@ class BmlEnginePreExecuteHook extends ComputationExecutorHook with Logging { ): String = { val props = engineExecutionContext.getProperties if (null != props && props.containsKey(GovernanceConstant.TASK_RESOURCES_STR)) { - val workDir = ComputationEngineUtils.getCurrentWorkDir val jobId = engineExecutionContext.getJobId props.get(GovernanceConstant.TASK_RESOURCES_STR) match { case resources: util.List[Object] => @@ -71,9 +70,7 @@ class BmlEnginePreExecuteHook extends ComputationExecutorHook with Logging { val fileName = resource.get(GovernanceConstant.TASK_RESOURCE_FILE_NAME_STR).toString val resourceId = resource.get(GovernanceConstant.TASK_RESOURCE_ID_STR).toString val version = resource.get(GovernanceConstant.TASK_RESOURCE_VERSION_STR).toString - val fullPath = - if (workDir.endsWith(seperator)) pathType + workDir + fileName - else pathType + workDir + seperator + fileName + val fullPath = fileName val response = Utils.tryCatch { bmlClient.downloadShareResource(processUser, resourceId, version, fullPath, true) } { diff --git a/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/conf/ComputationExecutorConf.scala b/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/conf/ComputationExecutorConf.scala index c072c32794..bcd423fd21 100644 --- a/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/conf/ComputationExecutorConf.scala +++ b/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/conf/ComputationExecutorConf.scala @@ -33,6 +33,12 @@ object ComputationExecutorConf { "Maximum number of tasks executed by the synchronization EC" ) + val PRINT_TASK_PARAMS_SKIP_KEYS = CommonVars( + "linkis.engineconn.print.task.params.skip.keys", + "jobId", + "skip to print params key at job logs" + ) + val ENGINE_PROGRESS_FETCH_INTERVAL = CommonVars( "wds.linkis.engineconn.progresss.fetch.interval-in-seconds", diff --git a/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/cs/CSResourceParser.scala b/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/cs/CSResourceParser.scala index f59adaadef..fe98e3328e 100644 --- a/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/cs/CSResourceParser.scala +++ b/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/cs/CSResourceParser.scala @@ -17,7 +17,10 @@ package org.apache.linkis.engineconn.computation.executor.cs +import org.apache.linkis.common.utils.Logging import org.apache.linkis.cs.client.service.CSResourceService +import org.apache.linkis.engineconn.common.conf.EngineConnConf +import org.apache.linkis.governance.common.utils.GovernanceConstant import org.apache.commons.lang3.StringUtils @@ -27,7 +30,7 @@ import java.util.regex.Pattern import scala.collection.JavaConverters._ import scala.collection.mutable.ArrayBuffer -class CSResourceParser { +class CSResourceParser extends Logging { private val pb = Pattern.compile("cs://[^\\s\"]+[$\\s]{0,1}", Pattern.CASE_INSENSITIVE) @@ -47,7 +50,6 @@ class CSResourceParser { nodeNameStr: String ): String = { - // TODO getBMLResource val bmlResourceList = CSResourceService.getInstance().getUpstreamBMLResource(contextIDValueStr, nodeNameStr) @@ -56,23 +58,25 @@ class CSResourceParser { val preFixNames = new ArrayBuffer[String]() val parsedNames = new ArrayBuffer[String]() + val prefixName = System.currentTimeMillis().toString + "_" preFixResourceNames.foreach { preFixResourceName => val resourceName = preFixResourceName.replace(PREFIX, "").trim val bmlResourceOption = bmlResourceList.asScala.find(_.getDownloadedFileName.equals(resourceName)) if (bmlResourceOption.isDefined) { + val replacementName = EngineConnConf.getEngineTmpDir + prefixName + resourceName val bmlResource = bmlResourceOption.get val map = new util.HashMap[String, Object]() - map.put("resourceId", bmlResource.getResourceId) - map.put("version", bmlResource.getVersion) - map.put("fileName", resourceName) + map.put(GovernanceConstant.TASK_RESOURCE_ID_STR, bmlResource.getResourceId) + map.put(GovernanceConstant.TASK_RESOURCE_VERSION_STR, bmlResource.getVersion) + map.put(GovernanceConstant.TASK_RESOURCE_FILE_NAME_STR, replacementName) parsedResources.add(map) preFixNames.append(preFixResourceName) - parsedNames.append(resourceName) + parsedNames.append(replacementName) + logger.warn(s"Replace cs file from {$preFixResourceName} to {$replacementName}") } - } - props.put("resources", parsedResources) + props.put(GovernanceConstant.TASK_RESOURCES_STR, parsedResources) StringUtils.replaceEach(code, preFixNames.toArray, parsedNames.toArray) } diff --git a/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/execute/ComputationExecutor.scala b/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/execute/ComputationExecutor.scala index ea80b625bd..bd6d44e4a6 100644 --- a/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/execute/ComputationExecutor.scala +++ b/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/execute/ComputationExecutor.scala @@ -22,6 +22,7 @@ import org.apache.linkis.common.log.LogUtils import org.apache.linkis.common.utils.{Logging, Utils} import org.apache.linkis.engineconn.acessible.executor.entity.AccessibleExecutor import org.apache.linkis.engineconn.acessible.executor.listener.event.{ + TaskLogUpdateEvent, TaskResponseErrorEvent, TaskStatusChangedEvent } @@ -40,7 +41,14 @@ import org.apache.linkis.governance.common.paser.CodeParser import org.apache.linkis.governance.common.protocol.task.{EngineConcurrentInfo, RequestTask} import org.apache.linkis.governance.common.utils.{JobUtils, LoggerUtils} import org.apache.linkis.manager.common.entity.enumeration.NodeStatus -import org.apache.linkis.manager.label.entity.engine.UserCreatorLabel +import org.apache.linkis.manager.label.entity.engine.{ + CodeLanguageLabel, + EngineType, + EngineTypeLabel, + RunType, + UserCreatorLabel +} +import org.apache.linkis.manager.label.utils.LabelUtil import org.apache.linkis.protocol.engine.JobProgressInfo import org.apache.linkis.scheduler.executer._ @@ -50,6 +58,8 @@ import org.apache.commons.lang3.exception.ExceptionUtils import java.util.concurrent.TimeUnit import java.util.concurrent.atomic.AtomicInteger +import scala.collection.JavaConverters._ + import com.google.common.cache.{Cache, CacheBuilder} abstract class ComputationExecutor(val outputPrintLimit: Int = 1000) @@ -132,6 +142,12 @@ abstract class ComputationExecutor(val outputPrintLimit: Int = 1000) override def close(): Unit = { if (null != lastTask) CLOSE_LOCKER.synchronized { + listenerBusContext.getEngineConnSyncListenerBus.postToAll( + TaskLogUpdateEvent( + lastTask.getTaskId, + LogUtils.generateERROR("EC exits unexpectedly and actively kills the task") + ) + ) killTask(lastTask.getTaskId) } else { @@ -169,9 +185,11 @@ abstract class ComputationExecutor(val outputPrintLimit: Int = 1000) Utils.tryFinally { transformTaskStatus(engineConnTask, ExecutionNodeStatus.Running) val engineExecutionContext = createEngineExecutionContext(engineConnTask) + + val engineCreationContext = EngineConnObject.getEngineCreationContext + var hookedCode = engineConnTask.getCode Utils.tryCatch { - val engineCreationContext = EngineConnObject.getEngineCreationContext ComputationExecutorHook.getComputationExecutorHooks.foreach(hook => { hookedCode = hook.beforeExecutorExecute(engineExecutionContext, engineCreationContext, hookedCode) @@ -182,12 +200,24 @@ abstract class ComputationExecutor(val outputPrintLimit: Int = 1000) } else { logger.info(s"hooked after code: $hookedCode ") } + + // task params log + // spark engine: at org.apache.linkis.engineplugin.spark.executor.SparkEngineConnExecutor.executeLine log special conf + Utils.tryAndWarn { + val engineType = LabelUtil.getEngineType(engineCreationContext.getLabels()) + EngineType.mapStringToEngineType(engineType) match { + case EngineType.HIVE | EngineType.TRINO => printTaskParamsLog(engineExecutionContext) + case _ => + } + } + val localPath = EngineConnConf.getLogDir engineExecutionContext.appendStdout( LogUtils.generateInfo( s"EngineConn local log path: ${DataWorkCloudApplication.getServiceInstance.toString} $localPath" ) ) + var response: ExecuteResponse = null val incomplete = new StringBuilder val codes = @@ -244,6 +274,11 @@ abstract class ComputationExecutor(val outputPrintLimit: Int = 1000) case s: SuccessExecuteResponse => succeedTasks.increase() s + case incompleteExecuteResponse: IncompleteExecuteResponse => + ErrorExecuteResponse( + s"The task cannot be an incomplete response ${incompleteExecuteResponse.message}", + null + ) case _ => response } response @@ -336,6 +371,30 @@ abstract class ComputationExecutor(val outputPrintLimit: Int = 1000) } } + /** + * job task log print task params info + * + * @param engineExecutorContext + * @return + * Unit + */ + + def printTaskParamsLog(engineExecutorContext: EngineExecutionContext): Unit = { + val sb = new StringBuilder + + EngineConnObject.getEngineCreationContext.getOptions.asScala.foreach({ case (key, value) => + // skip log jobId because it corresponding jobid when the ec created + if (!ComputationExecutorConf.PRINT_TASK_PARAMS_SKIP_KEYS.getValue.contains(key)) { + sb.append(s"${key}=${value.toString}\n") + } + }) + + sb.append("\n") + engineExecutorContext.appendStdout( + LogUtils.generateInfo(s"Your job exec with configs:\n${sb.toString()}\n") + ) + } + def transformTaskStatus(task: EngineConnTask, newStatus: ExecutionNodeStatus): Unit = { val oriStatus = task.getStatus logger.info(s"task ${task.getTaskId} from status $oriStatus to new status $newStatus") diff --git a/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/metrics/ComputationEngineConnMetrics.scala b/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/metrics/ComputationEngineConnMetrics.scala index f96896f557..4446bdc672 100644 --- a/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/metrics/ComputationEngineConnMetrics.scala +++ b/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/metrics/ComputationEngineConnMetrics.scala @@ -91,4 +91,12 @@ object ComputationEngineConnMetrics { getTotalBusyTimeMills(nodeStatus) + getTotalIdleTimeMills(nodeStatus) def getUnlockToShutdownDurationMills(): Long = unlockToShutdownDurationMills.get() + + def getLastUnlockTimestamp(nodeStatus: NodeStatus): Long = { + nodeStatus match { + case NodeStatus.Unlock => lastUnlockTimeMills + case _ => 0 + } + } + } diff --git a/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/service/DefaultNodeHeartbeatMsgManager.scala b/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/service/DefaultNodeHeartbeatMsgManager.scala index 010ced97fd..e5d74282de 100644 --- a/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/service/DefaultNodeHeartbeatMsgManager.scala +++ b/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/service/DefaultNodeHeartbeatMsgManager.scala @@ -66,6 +66,10 @@ class DefaultNodeHeartbeatMsgManager extends NodeHeartbeatMsgManager with Loggin ECConstants.EC_TOTAL_LOCK_TIME_MILLS_KEY, ComputationEngineConnMetrics.getTotalLockTimeMills(status).asInstanceOf[Object] ) + msgMap.put( + ECConstants.EC_LAST_UNLOCK_TIMESTAMP, + ComputationEngineConnMetrics.getLastUnlockTimestamp(status).asInstanceOf[Object] + ) case _ => } val engineParams = EngineConnObject.getEngineCreationContext.getOptions diff --git a/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/service/TaskExecutionServiceImpl.scala b/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/service/TaskExecutionServiceImpl.scala index ca1887e746..6b4fc64fe6 100644 --- a/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/service/TaskExecutionServiceImpl.scala +++ b/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/service/TaskExecutionServiceImpl.scala @@ -422,31 +422,51 @@ class TaskExecutionServiceImpl ): Future[_] = { val sleepInterval = ComputationExecutorConf.ENGINE_PROGRESS_FETCH_INTERVAL.getValue scheduler.submit(new Runnable { - override def run(): Unit = Utils.tryAndWarn { + override def run(): Unit = { Utils.tryQuietly(Thread.sleep(TimeUnit.MILLISECONDS.convert(1, TimeUnit.SECONDS))) while (null != taskFuture && !taskFuture.isDone) { - if ( - ExecutionNodeStatus.isCompleted(task.getStatus) || ExecutionNodeStatus - .isRunning(task.getStatus) - ) { - val progressResponse = taskProgress(task.getTaskId) - val resourceResponse = buildResourceMap(task) - val extraInfoMap = buildExtraInfoMap(task) - // todo add other info - val resourceMap = - if (null != resourceResponse) resourceResponse.getResourceMap else null - - val respRunningInfo: ResponseTaskRunningInfo = new ResponseTaskRunningInfo( - progressResponse.execId, - progressResponse.progress, - progressResponse.progressInfo, - resourceMap, - extraInfoMap - ) - - sendToEntrance(task, respRunningInfo) - Thread.sleep(TimeUnit.MILLISECONDS.convert(sleepInterval, TimeUnit.SECONDS)) + if (!ExecutionNodeStatus.isCompleted(task.getStatus)) { + Utils.tryAndWarn { + val progressResponse = Utils.tryCatch(taskProgress(task.getTaskId)) { + case e: Exception => + logger.info("Failed to get progress", e) + null + } + val resourceResponse = Utils.tryCatch(buildResourceMap(task)) { case e: Exception => + logger.info("Failed to get resource", e) + null + } + val extraInfoMap = Utils.tryCatch(buildExtraInfoMap(task)) { case e: Exception => + logger.info("Failed to get extra info ", e) + null + } + val resourceMap = + if (null != resourceResponse) resourceResponse.getResourceMap else null + + /** + * It is guaranteed that there must be progress the progress must be greater than or + * equal to 0.1 + */ + val newProgressResponse = if (null == progressResponse) { + ResponseTaskProgress(task.getTaskId, 0.1f, null) + } else if (progressResponse.progress < 0.1f) { + ResponseTaskProgress(task.getTaskId, 0.1f, progressResponse.progressInfo) + } else { + progressResponse + } + val respRunningInfo: ResponseTaskRunningInfo = new ResponseTaskRunningInfo( + newProgressResponse.execId, + newProgressResponse.progress, + newProgressResponse.progressInfo, + resourceMap, + extraInfoMap + ) + sendToEntrance(task, respRunningInfo) + } } + Utils.tryQuietly( + Thread.sleep(TimeUnit.MILLISECONDS.convert(sleepInterval, TimeUnit.SECONDS)) + ) } } }) @@ -499,7 +519,7 @@ class TaskExecutionServiceImpl } override def taskProgress(taskID: String): ResponseTaskProgress = { - var response = ResponseTaskProgress(taskID, 0, null) + var response = ResponseTaskProgress(taskID, 0.01f, null) if (StringUtils.isBlank(taskID)) return response val executor = taskIdCache.getIfPresent(taskID) if (null != executor) { @@ -514,11 +534,9 @@ class TaskExecutionServiceImpl ResponseTaskProgress(taskID, progress, executor.getProgressInfo(taskID)) ) } - } else { - response = ResponseTaskProgress(taskID, -1, null) } } else { - logger.error(s"Executor of taskId : $taskID is not cached.") + logger.info(s"Executor of taskId : $taskID is not cached.") } response } @@ -606,14 +624,14 @@ class TaskExecutionServiceImpl logger.warn("Unknown event : " + BDPJettyServerHelper.gson.toJson(event)) } - override def onLogUpdate(logUpdateEvent: TaskLogUpdateEvent): Unit = { + override def onLogUpdate(logUpdateEvent: TaskLogUpdateEvent): Unit = Utils.tryAndWarn { if (EngineConnConf.ENGINE_PUSH_LOG_TO_ENTRANCE.getValue) { if (null != logUpdateEvent && StringUtils.isNotBlank(logUpdateEvent.taskId)) { val task = getTaskByTaskId(logUpdateEvent.taskId) if (null != task) { sendToEntrance(task, ResponseTaskLog(logUpdateEvent.taskId, logUpdateEvent.log)) } else { - logger.error("Task cannot null! logupdateEvent: " + logUpdateEvent.taskId) + logger.warn("Task cannot null! logupdateEvent: " + logUpdateEvent.taskId) } } else if (null != lastTask) { val executor = executorManager.getReportExecutor @@ -663,32 +681,33 @@ class TaskExecutionServiceImpl } } - override def onProgressUpdate(taskProgressUpdateEvent: TaskProgressUpdateEvent): Unit = { - if (EngineConnConf.ENGINE_PUSH_LOG_TO_ENTRANCE.getValue) { - val task = getTaskByTaskId(taskProgressUpdateEvent.taskId) - if (null != task) { - val resourceResponse = buildResourceMap(task) - val extraInfoMap = buildExtraInfoMap(task) + override def onProgressUpdate(taskProgressUpdateEvent: TaskProgressUpdateEvent): Unit = + Utils.tryAndWarn { + if (EngineConnConf.ENGINE_PUSH_LOG_TO_ENTRANCE.getValue) { + val task = getTaskByTaskId(taskProgressUpdateEvent.taskId) + if (null != task) { + val resourceResponse = buildResourceMap(task) + val extraInfoMap = buildExtraInfoMap(task) - val resourceMap = if (null != resourceResponse) resourceResponse.getResourceMap else null + val resourceMap = if (null != resourceResponse) resourceResponse.getResourceMap else null - val respRunningInfo: ResponseTaskRunningInfo = new ResponseTaskRunningInfo( - taskProgressUpdateEvent.taskId, - taskProgressUpdateEvent.progress, - taskProgressUpdateEvent.progressInfo, - resourceMap, - extraInfoMap - ) + val respRunningInfo: ResponseTaskRunningInfo = new ResponseTaskRunningInfo( + taskProgressUpdateEvent.taskId, + taskProgressUpdateEvent.progress, + taskProgressUpdateEvent.progressInfo, + resourceMap, + extraInfoMap + ) - sendToEntrance(task, respRunningInfo) - } else { - logger.error( - "Task cannot null! taskProgressUpdateEvent : " + ComputationEngineUtils.GSON - .toJson(taskProgressUpdateEvent) - ) + sendToEntrance(task, respRunningInfo) + } else { + logger.error( + "Task cannot null! taskProgressUpdateEvent : " + ComputationEngineUtils.GSON + .toJson(taskProgressUpdateEvent) + ) + } } } - } override def onResultSetCreated(taskResultCreateEvent: TaskResultCreateEvent): Unit = { logger.info(s"start to deal result event ${taskResultCreateEvent.taskId}") @@ -714,7 +733,7 @@ class TaskExecutionServiceImpl if (null != executor) { executor.getTaskById(taskId) } else { - logger.error(s"Executor of taskId : $taskId is not cached.") + logger.warn(s"Executor of taskId : $taskId is not cached.") null } } diff --git a/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/upstream/handler/ECTaskKillHandler.scala b/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/upstream/handler/ECTaskKillHandler.scala index d06e8ac077..15e70315e3 100644 --- a/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/upstream/handler/ECTaskKillHandler.scala +++ b/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/upstream/handler/ECTaskKillHandler.scala @@ -39,10 +39,10 @@ class ECTaskKillHandler extends MonitorHandler with Logging { while (elements.hasNext) { val element = elements.next Utils.tryCatch { - doKill(element) logger.error( s"ERROR: entrance : ${element.getUpstreamConnection().getUpstreamServiceInstanceName()} lose connect, will kill job : ${element.getKey()}" ) + doKill(element) } { t => logger.error("Failed to kill job: " + element.getKey, t) } diff --git a/linkis-computation-governance/linkis-engineconn/linkis-engineconn-common/src/main/scala/org/apache/linkis/engineconn/common/conf/EngineConnConf.scala b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-common/src/main/scala/org/apache/linkis/engineconn/common/conf/EngineConnConf.scala index eefc69f5bb..43f7ab4446 100644 --- a/linkis-computation-governance/linkis-engineconn/linkis-engineconn-common/src/main/scala/org/apache/linkis/engineconn/common/conf/EngineConnConf.scala +++ b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-common/src/main/scala/org/apache/linkis/engineconn/common/conf/EngineConnConf.scala @@ -61,6 +61,8 @@ object EngineConnConf { val ENGINE_CONN_LOCAL_LOG_DIRS_KEY = CommonVars("wds.linkis.engine.logs.dir.key", "LOG_DIRS") + val ENGINE_CONN_LOCAL_TMP_DIR = CommonVars("wds.linkis.engine.tmp.dir", "TEMP_DIRS") + val ENGINE_CONN_CREATION_WAIT_TIME = CommonVars("wds.linkis.engine.connector.init.time", new TimeType("8m")) @@ -84,6 +86,8 @@ object EngineConnConf { def getWorkHome: String = System.getenv(ENGINE_CONN_LOCAL_PATH_PWD_KEY.getValue) + def getEngineTmpDir: String = System.getenv(ENGINE_CONN_LOCAL_TMP_DIR.getValue) + def getLogDir: String = { val logDir = System.getenv(ENGINE_CONN_LOCAL_LOG_DIRS_KEY.getValue) if (StringUtils.isNotEmpty(logDir)) logDir else new File(getWorkHome, "logs").getPath diff --git a/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/org/apache/linkis/engineconn/acessible/executor/conf/AccessibleExecutorConfiguration.scala b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/org/apache/linkis/engineconn/acessible/executor/conf/AccessibleExecutorConfiguration.scala index 95a01202e8..40cf314853 100644 --- a/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/org/apache/linkis/engineconn/acessible/executor/conf/AccessibleExecutorConfiguration.scala +++ b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/org/apache/linkis/engineconn/acessible/executor/conf/AccessibleExecutorConfiguration.scala @@ -37,7 +37,7 @@ object AccessibleExecutorConfiguration { val ENGINECONN_LOG_SEND_SIZE = CommonVars[Int]("wds.linkis.engineconn.log.send.cache.size", 300) val ENGINECONN_MAX_FREE_TIME = - CommonVars("wds.linkis.engineconn.max.free.time", new TimeType("30m")) + CommonVars("wds.linkis.engineconn.max.free.time", new TimeType("10m")) val ENGINECONN_LOCK_CHECK_INTERVAL = CommonVars("wds.linkis.engineconn.lock.free.interval", new TimeType("3m")) diff --git a/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/conf/EntranceSpringConfiguration.java b/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/conf/EntranceSpringConfiguration.java index cf520c3823..86b1a91f7a 100644 --- a/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/conf/EntranceSpringConfiguration.java +++ b/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/conf/EntranceSpringConfiguration.java @@ -150,7 +150,7 @@ public EntranceInterceptor[] entranceInterceptors() { new ScalaCodeInterceptor(), new SQLLimitEntranceInterceptor(), new CommentInterceptor(), - new SetTenantLabelInterceptor(), + // new SetTenantLabelInterceptor(), new UserCreatorIPCheckInterceptor() }; } diff --git a/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/errorcode/EntranceErrorCodeSummary.java b/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/errorcode/EntranceErrorCodeSummary.java index b5f90e3070..51a522d3d2 100644 --- a/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/errorcode/EntranceErrorCodeSummary.java +++ b/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/errorcode/EntranceErrorCodeSummary.java @@ -62,6 +62,11 @@ public enum EntranceErrorCodeSummary implements LinkisErrorCode { INVALID_RESULTSETS(20053, "Invalid resultsets, cannot use cache(结果集无效，无法使用 cache)"), SUBMITTING_QUERY_FAILED(30009, "Submitting the query failed(提交查询失败)!"), + + SUBMIT_CODE_ISEMPTY( + 30010, + "Submitting the execution code, after code preprocessing, the real execution code is empty, please check the executed code(提交的执行代码，经过预处理后为空，请检查执行的代码是否为空或则只有注解)!"), + QUERY_STATUS_FAILED(50081, "Query from jobHistory status failed(从 jobHistory 状态查询失败)"), GET_QUERY_RESPONSE(50081, "Get query response incorrectly(获取查询响应结果不正确)"), QUERY_TASKID_ERROR(50081, "Query task of taskId:{0} error(查询任务id：{}的任务出错)"), diff --git a/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/restful/EntranceConsumerRestfulApi.java b/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/restful/EntranceConsumerRestfulApi.java new file mode 100644 index 0000000000..424e7ca170 --- /dev/null +++ b/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/restful/EntranceConsumerRestfulApi.java @@ -0,0 +1,88 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.entrance.restful; + +import org.apache.linkis.common.conf.Configuration; +import org.apache.linkis.entrance.EntranceServer; +import org.apache.linkis.scheduler.queue.ConsumerManager; +import org.apache.linkis.server.Message; +import org.apache.linkis.server.utils.ModuleUserUtils; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestMethod; +import org.springframework.web.bind.annotation.RequestParam; +import org.springframework.web.bind.annotation.RestController; + +import javax.servlet.http.HttpServletRequest; + +import io.swagger.annotations.Api; +import io.swagger.annotations.ApiOperation; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +@Api(tags = "entrance lable manager") +@RestController +@RequestMapping(path = "/entrance/operation/consumer") +public class EntranceConsumerRestfulApi { + + private EntranceServer entranceServer; + + private static final Logger logger = LoggerFactory.getLogger(EntranceConsumerRestfulApi.class); + + @Autowired + public void setEntranceServer(EntranceServer entranceServer) { + this.entranceServer = entranceServer; + } + + @ApiOperation(value = "kill-consumer", notes = "kill consumer", response = Message.class) + @RequestMapping(path = "/kill", method = RequestMethod.GET) + public Message killConsumer( + HttpServletRequest req, @RequestParam(value = "groupName") String groupName) { + String operationUser = ModuleUserUtils.getOperationUser(req, "kill consumer"); + if (Configuration.isNotAdmin(operationUser)) { + return Message.error("only admin can do this"); + } + logger.info("user {} to kill consumer {}", operationUser, groupName); + ConsumerManager consumerManager = + entranceServer + .getEntranceContext() + .getOrCreateScheduler() + .getSchedulerContext() + .getOrCreateConsumerManager(); + consumerManager.destroyConsumer(groupName); + logger.info("user {} finished to kill consumer {}", operationUser, groupName); + return Message.ok(); + } + + @ApiOperation(value = "consumer-info", notes = "list consumers info", response = Message.class) + @RequestMapping(path = "/info", method = RequestMethod.GET) + public Message countConsumer(HttpServletRequest req) { + String operationUser = ModuleUserUtils.getOperationUser(req, "kill consumer"); + if (Configuration.isNotAdmin(operationUser)) { + return Message.error("only admin can do this"); + } + ConsumerManager consumerManager = + entranceServer + .getEntranceContext() + .getOrCreateScheduler() + .getSchedulerContext() + .getOrCreateConsumerManager(); + return Message.ok().data("consumerNum", consumerManager.listConsumers().length); + } +} diff --git a/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/restful/EntranceLabelRestfulApi.java b/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/restful/EntranceLabelRestfulApi.java index 841a6a3fb0..0737e25ed8 100644 --- a/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/restful/EntranceLabelRestfulApi.java +++ b/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/restful/EntranceLabelRestfulApi.java @@ -24,6 +24,7 @@ import org.apache.linkis.manager.label.constant.LabelKeyConstant; import org.apache.linkis.manager.label.constant.LabelValueConstant; import org.apache.linkis.protocol.label.InsLabelRefreshRequest; +import org.apache.linkis.protocol.label.InsLabelRemoveRequest; import org.apache.linkis.rpc.Sender; import org.apache.linkis.scheduler.SchedulerContext; import org.apache.linkis.server.Message; @@ -57,6 +58,8 @@ public void setEntranceServer(EntranceServer entranceServer) { this.entranceServer = entranceServer; } + private static Boolean offlineFlag = false; + @ApiOperation(value = "update", notes = "update route label", response = Message.class) @ApiOperationSupport(ignoreParameters = {"jsonNode"}) @RequestMapping(path = "/update", method = RequestMethod.POST) @@ -82,12 +85,15 @@ public Message updateRouteLabel(HttpServletRequest req, @RequestBody JsonNode js public Message updateRouteLabel(HttpServletRequest req) { ModuleUserUtils.getOperationUser(req, "markoffline"); Map labels = new HashMap(); - logger.info("Prepare to modify the routelabel of entry to offline"); + logger.info("Prepare to modify the routelabel of entrance to offline"); labels.put(LabelKeyConstant.ROUTE_KEY, LabelValueConstant.OFFLINE_VALUE); InsLabelRefreshRequest insLabelRefreshRequest = new InsLabelRefreshRequest(); insLabelRefreshRequest.setLabels(labels); insLabelRefreshRequest.setServiceInstance(Sender.getThisServiceInstance()); InstanceLabelClient.getInstance().refreshLabelsToInstance(insLabelRefreshRequest); + synchronized (offlineFlag) { + offlineFlag = true; + } logger.info("Finished to modify the routelabel of entry to offline"); logger.info("Prepare to update all not execution task instances to empty string"); @@ -101,4 +107,29 @@ public Message updateRouteLabel(HttpServletRequest req) { return Message.ok(); } + + @ApiOperation( + value = "backonline", + notes = "from offline status to recover", + response = Message.class) + @RequestMapping(path = "/backonline", method = RequestMethod.GET) + public Message backOnline(HttpServletRequest req) { + ModuleUserUtils.getOperationUser(req, "backonline"); + logger.info("Prepare to modify the routelabel of entrance to remove offline"); + InsLabelRemoveRequest insLabelRemoveRequest = new InsLabelRemoveRequest(); + insLabelRemoveRequest.setServiceInstance(Sender.getThisServiceInstance()); + InstanceLabelClient.getInstance().removeLabelsFromInstance(insLabelRemoveRequest); + synchronized (offlineFlag) { + offlineFlag = false; + } + logger.info("Finished to backonline"); + return Message.ok(); + } + + @ApiOperation(value = "isOnline", notes = "entrance isOnline", response = Message.class) + @RequestMapping(path = "/isOnline", method = RequestMethod.GET) + public Message isOnline(HttpServletRequest req) { + logger.info("Whether Entrance is online: {}", !offlineFlag); + return Message.ok().data("isOnline", !offlineFlag); + } } diff --git a/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/restful/EntranceMetricRestfulApi.java b/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/restful/EntranceMetricRestfulApi.java index 7d36df8fec..7b487352d5 100644 --- a/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/restful/EntranceMetricRestfulApi.java +++ b/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/restful/EntranceMetricRestfulApi.java @@ -20,8 +20,7 @@ import org.apache.linkis.common.conf.Configuration; import org.apache.linkis.entrance.EntranceServer; import org.apache.linkis.entrance.execute.EntranceJob; -import org.apache.linkis.manager.label.entity.engine.EngineTypeLabel; -import org.apache.linkis.manager.label.utils.LabelUtil; +import org.apache.linkis.entrance.scheduler.CreatorECTypeDefaultConf; import org.apache.linkis.server.Message; import org.apache.linkis.server.utils.ModuleUserUtils; @@ -67,7 +66,7 @@ public Message taskinfo( HttpServletRequest req, @RequestParam(value = "user", required = false) String user, @RequestParam(value = "creator", required = false) String creator, - @RequestParam(value = "engineTypeLabel", required = false) String engineTypeLabelValue) { + @RequestParam(value = "ecType", required = false) String ecType) { String userName = ModuleUserUtils.getOperationUser(req, "taskinfo"); String queryUser = user; if (Configuration.isNotAdmin(userName)) { @@ -83,23 +82,12 @@ public Message taskinfo( } else if (StringUtils.isBlank(creator)) { filterWords = queryUser; } - EntranceJob[] undoneTasks = entranceServer.getAllUndoneTask(filterWords); - int taskNumber = 0; + EntranceJob[] undoneTasks = entranceServer.getAllUndoneTask(filterWords, ecType); int runningNumber = 0; int queuedNumber = 0; if (null != undoneTasks) { for (EntranceJob task : undoneTasks) { - if (StringUtils.isNotBlank(engineTypeLabelValue)) { - EngineTypeLabel engineTypeLabel = - LabelUtil.getEngineTypeLabel(task.getJobRequest().getLabels()); - // Task types do not match, do not count - if (null == engineTypeLabel - || !engineTypeLabelValue.equalsIgnoreCase(engineTypeLabel.getStringValue())) { - continue; - } - } - taskNumber++; if (task.isRunning()) { runningNumber++; } else { @@ -107,17 +95,25 @@ public Message taskinfo( } } } - return Message.ok("success") - .data("taskNumber", taskNumber) - .data("runningNumber", runningNumber) - .data("queuedNumber", queuedNumber); + Message resp = + Message.ok("success") + .data("taskNumber", undoneTasks.length) + .data("runningNumber", runningNumber) + .data("queuedNumber", queuedNumber); + if (StringUtils.isNoneBlank(creator, ecType)) { + int creatorECTypeMaxRunningJobs = + CreatorECTypeDefaultConf.getCreatorECTypeMaxRunningJobs(creator, ecType); + resp.data("creatorECTypeMaxRunningJobs", creatorECTypeMaxRunningJobs); + resp.data("limitExceeded", runningNumber > creatorECTypeMaxRunningJobs); + } + return resp; } - @ApiOperation(value = "Status", notes = "get running task number ", response = Message.class) + @ApiOperation(value = "runningtask", notes = "get running task number ", response = Message.class) @RequestMapping(path = "/runningtask", method = RequestMethod.GET) - public Message status(HttpServletRequest req) { + public Message runningtask(HttpServletRequest req) { ModuleUserUtils.getOperationUser(req, "runningtask"); - EntranceJob[] undoneTasks = entranceServer.getAllUndoneTask(""); + EntranceJob[] undoneTasks = entranceServer.getAllUndoneTask("", null); Boolean isCompleted = false; if (null == undoneTasks || undoneTasks.length < 1) { isCompleted = true; diff --git a/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/restful/EntranceRestfulApi.java b/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/restful/EntranceRestfulApi.java index cf9cd33653..dfab300ab4 100644 --- a/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/restful/EntranceRestfulApi.java +++ b/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/restful/EntranceRestfulApi.java @@ -114,15 +114,6 @@ public Message execute(HttpServletRequest req, @RequestBody Map JobRequest jobReq = ((EntranceJob) job).getJobRequest(); Long jobReqId = jobReq.getId(); ModuleUserUtils.getOperationUser(req, "execute task,id: " + jobReqId); - pushLog( - LogUtils.generateInfo( - "You have submitted a new job, script code (after variable substitution) is"), - job); - pushLog( - "************************************SCRIPT CODE************************************", job); - pushLog(jobReq.getExecutionCode(), job); - pushLog( - "************************************SCRIPT CODE************************************", job); String execID = ZuulEntranceUtils.generateExecID( job.getId(), @@ -166,15 +157,6 @@ public Message submit(HttpServletRequest req, @RequestBody Map j JobRequest jobRequest = ((EntranceJob) job).getJobRequest(); Long jobReqId = jobRequest.getId(); ModuleUserUtils.getOperationUser(req, "submit jobReqId: " + jobReqId); - pushLog( - LogUtils.generateInfo( - "You have submitted a new job, script code (after variable substitution) is"), - job); - pushLog( - "************************************SCRIPT CODE************************************", job); - pushLog(jobRequest.getExecutionCode(), job); - pushLog( - "************************************SCRIPT CODE************************************", job); pushLog( LogUtils.generateInfo( "Your job is accepted, jobID is " @@ -809,9 +791,9 @@ public Message killJobs( logger.error("kill job {} failed ", job.get().getId(), t); message = Message.error( - "An exception occurred while killing the job, kill failed(kill job的时候出现了异常，kill失败)"); + "An exception occurred while killing the job, kill failed(kill job的时候出现了异常，kill失败)", + t); message.setMethod("/api/entrance/" + id + "/kill"); - message.setStatus(1); } } messages.add(message); @@ -937,10 +919,11 @@ public Message kill( logger.error("kill job {} failed ", job.get().getId(), t); message = Message.error( - "An exception occurred while killing the job, kill failed(kill job的时候出现了异常，kill失败)" - + "message: " - + t.getMessage()); + "An exception occurred while killing the job, kill failed(kill job的时候出现了异常，kill失败) with error:" + + t.getMessage(), + t); message.setMethod("/api/entrance/" + id + "/kill"); + message.setStatus(1); } } return message; diff --git a/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/scheduler/CreatorECTypeDefaultConf.java b/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/scheduler/CreatorECTypeDefaultConf.java new file mode 100644 index 0000000000..5a91c71a11 --- /dev/null +++ b/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/scheduler/CreatorECTypeDefaultConf.java @@ -0,0 +1,93 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.entrance.scheduler; + +import org.apache.linkis.common.conf.Configuration; +import org.apache.linkis.entrance.conf.EntranceConfiguration; +import org.apache.linkis.entrance.utils.EntranceUtils; +import org.apache.linkis.governance.common.protocol.conf.RequestQueryEngineConfig; +import org.apache.linkis.governance.common.protocol.conf.ResponseQueryConfig; +import org.apache.linkis.manager.label.entity.engine.EngineTypeLabel; +import org.apache.linkis.manager.label.entity.engine.UserCreatorLabel; +import org.apache.linkis.rpc.Sender; + +import org.apache.commons.lang3.StringUtils; + +import java.util.concurrent.TimeUnit; + +import scala.Tuple2; + +import com.google.common.cache.CacheBuilder; +import com.google.common.cache.CacheLoader; +import com.google.common.cache.LoadingCache; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class CreatorECTypeDefaultConf { + + private static final Logger logger = LoggerFactory.getLogger(CreatorECTypeDefaultConf.class); + + public static Sender confSender = + Sender.getSender( + Configuration.CLOUD_CONSOLE_CONFIGURATION_SPRING_APPLICATION_NAME().getValue()); + + private static LoadingCache confCache = + CacheBuilder.newBuilder() + .maximumSize(1000) + .expireAfterWrite( + (long) EntranceConfiguration.ENTRANCE_CREATOR_JOB_LIMIT_CONF_CACHE().getValue(), + TimeUnit.MINUTES) + .build( + new CacheLoader() { + @Override + public Integer load(String key) throws Exception { + Tuple2 tuple2 = + EntranceUtils.fromKeyGetLabels(key); + RequestQueryEngineConfig requestQueryEngineConfig = + new RequestQueryEngineConfig(tuple2._1, tuple2._2(), null); + int jobLimit = + (int) EntranceConfiguration.ENTRANCE_CREATOR_JOB_LIMIT().getValue(); + try { + Object response = confSender.ask(requestQueryEngineConfig); + if (response instanceof ResponseQueryConfig) { + jobLimit = + (int) + EntranceConfiguration.ENTRANCE_CREATOR_JOB_LIMIT() + .getValue(((ResponseQueryConfig) response).getKeyAndValue()); + } + } catch (Exception e) { + logger.warn("Failed to get key {} from conf", key, e); + } + return jobLimit; + } + }); + + public static int getCreatorECTypeMaxRunningJobs(String creator, String ecType) { + int jobLimit = (int) EntranceConfiguration.ENTRANCE_CREATOR_JOB_LIMIT().getValue(); + if (StringUtils.isNoneBlank(creator, ecType)) { + try { + String key = EntranceUtils.getDefaultCreatorECTypeKey(creator, ecType); + jobLimit = confCache.get(key); + } catch (Exception e) { + logger.warn("Failed to get key creator {} ecType {} from cache", creator, ecType, e); + } + } + int entranceNumber = EntranceUtils.getRunningEntranceNumber(); + return jobLimit / entranceNumber; + } +} diff --git a/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/server/DefaultEntranceServer.java b/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/server/DefaultEntranceServer.java index 94531cd5fe..7c38d27947 100644 --- a/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/server/DefaultEntranceServer.java +++ b/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/server/DefaultEntranceServer.java @@ -104,7 +104,7 @@ private void shutdownEntrance(ContextClosedEvent event) { } logger.warn("Entrance exit to stop all job"); - EntranceJob[] allUndoneTask = getAllUndoneTask(null); + EntranceJob[] allUndoneTask = getAllUndoneTask(null, null); if (null != allUndoneTask) { for (EntranceJob job : allUndoneTask) { job.onFailure( diff --git a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/EntranceServer.scala b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/EntranceServer.scala index b023065ee4..a610d524b2 100644 --- a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/EntranceServer.scala +++ b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/EntranceServer.scala @@ -19,7 +19,6 @@ package org.apache.linkis.entrance import org.apache.linkis.common.ServiceInstance import org.apache.linkis.common.exception.{ErrorException, LinkisException, LinkisRuntimeException} -import org.apache.linkis.common.io.FsPath import org.apache.linkis.common.log.LogUtils import org.apache.linkis.common.utils.{Logging, Utils} import org.apache.linkis.entrance.conf.EntranceConfiguration @@ -29,7 +28,7 @@ import org.apache.linkis.entrance.errorcode.EntranceErrorCodeSummary._ import org.apache.linkis.entrance.exception.{EntranceErrorException, SubmitFailedException} import org.apache.linkis.entrance.execute.EntranceJob import org.apache.linkis.entrance.job.EntranceExecutionJob -import org.apache.linkis.entrance.log.{Cache, CacheLogWriter, HDFSCacheLogWriter, LogReader} +import org.apache.linkis.entrance.log.LogReader import org.apache.linkis.entrance.parser.ParserUtils import org.apache.linkis.entrance.timeout.JobTimeoutManager import org.apache.linkis.entrance.utils.JobHistoryHelper @@ -44,16 +43,14 @@ import org.apache.linkis.rpc.Sender import org.apache.linkis.rpc.conf.RPCConfiguration import org.apache.linkis.scheduler.queue.{Job, SchedulerEventState} import org.apache.linkis.server.conf.ServerConfiguration -import org.apache.linkis.storage.utils.StorageUtils import org.apache.commons.lang3.StringUtils import org.apache.commons.lang3.exception.ExceptionUtils -import org.springframework.beans.BeanUtils - import java.{lang, util} import java.text.{MessageFormat, SimpleDateFormat} import java.util.Date +import java.util.concurrent.TimeUnit import scala.collection.JavaConverters._ @@ -63,6 +60,8 @@ abstract class EntranceServer extends Logging { private val jobTimeoutManager: JobTimeoutManager = new JobTimeoutManager() + private val timeoutCheck = EntranceConfiguration.ENABLE_JOB_TIMEOUT_CHECK.getValue + def init(): Unit def getName: String @@ -126,6 +125,17 @@ abstract class EntranceServer extends Logging { * this to trigger JobListener.onJobinit() */ Utils.tryAndWarn(job.getJobListener.foreach(_.onJobInited(job))) + if (logger.isDebugEnabled()) { + logger.debug( + s"After code preprocessing, the real execution code is:${jobRequest.getExecutionCode}" + ) + } + if (StringUtils.isBlank(jobRequest.getExecutionCode)) { + throw new SubmitFailedException( + SUBMIT_CODE_ISEMPTY.getErrorCode, + SUBMIT_CODE_ISEMPTY.getErrorDesc + ) + } getEntranceContext.getOrCreateScheduler().submit(job) val msg = LogUtils.generateInfo( s"Job with jobId : ${jobRequest.getId} and execID : ${job.getId()} submitted " @@ -135,7 +145,7 @@ abstract class EntranceServer extends Logging { job match { case entranceJob: EntranceJob => entranceJob.getJobRequest.setReqId(job.getId()) - if (jobTimeoutManager.timeoutCheck && JobTimeoutManager.hasTimeoutLabel(entranceJob)) { + if (timeoutCheck && JobTimeoutManager.hasTimeoutLabel(entranceJob)) { jobTimeoutManager.add(job.getId(), entranceJob) } entranceJob.getLogListener.foreach(_.onLogUpdate(entranceJob, msg)) @@ -492,14 +502,14 @@ abstract class EntranceServer extends Logging { job.setProgressListener(getEntranceContext.getOrCreatePersistenceManager()) job.setJobListener(getEntranceContext.getOrCreatePersistenceManager()) job match { - case entranceJob: EntranceJob => { + case entranceJob: EntranceJob => entranceJob.setEntranceListenerBus(getEntranceContext.getOrCreateEventListenerBus) - } case _ => } Utils.tryCatch { - if (logAppender.length() > 0) + if (logAppender.length() > 0) { job.getLogListener.foreach(_.onLogUpdate(job, logAppender.toString.trim)) + } } { t => logger.error("Failed to write init JobRequest log, reason: ", t) } @@ -510,6 +520,17 @@ abstract class EntranceServer extends Logging { * this to trigger JobListener.onJobinit() */ Utils.tryAndWarn(job.getJobListener.foreach(_.onJobInited(job))) + if (logger.isDebugEnabled()) { + logger.debug( + s"After code preprocessing, the real execution code is:${jobRequest.getExecutionCode}" + ) + } + if (StringUtils.isBlank(jobRequest.getExecutionCode)) { + throw new SubmitFailedException( + SUBMIT_CODE_ISEMPTY.getErrorCode, + SUBMIT_CODE_ISEMPTY.getErrorDesc + ) + } getEntranceContext.getOrCreateScheduler().submit(job) val msg = LogUtils.generateInfo( s"Job with jobId : ${jobRequest.getId} and execID : ${job.getId()} submitted, success to failover" @@ -519,8 +540,9 @@ abstract class EntranceServer extends Logging { job match { case entranceJob: EntranceJob => entranceJob.getJobRequest.setReqId(job.getId()) - if (jobTimeoutManager.timeoutCheck && JobTimeoutManager.hasTimeoutLabel(entranceJob)) + if (timeoutCheck && JobTimeoutManager.hasTimeoutLabel(entranceJob)) { jobTimeoutManager.add(job.getId(), entranceJob) + } entranceJob.getLogListener.foreach(_.onLogUpdate(entranceJob, msg)) case _ => } @@ -621,6 +643,68 @@ abstract class EntranceServer extends Logging { logger.info(s"job ${jobRequest.getId} success to initialize the properties") } + def getAllUndoneTask(filterWords: String, ecType: String = null): Array[EntranceJob] = { + val consumers = getEntranceContext + .getOrCreateScheduler() + .getSchedulerContext + .getOrCreateConsumerManager + .listConsumers() + .toSet + val filterConsumer = if (StringUtils.isNotBlank(filterWords)) { + if (StringUtils.isNotBlank(ecType)) { + consumers.filter(consumer => + consumer.getGroup.getGroupName.contains(filterWords) && consumer.getGroup.getGroupName + .contains(ecType) + ) + } else { + consumers.filter(_.getGroup.getGroupName.contains(filterWords)) + } + } else { + consumers + } + filterConsumer + .flatMap { consumer => + consumer.getRunningEvents ++ consumer.getConsumeQueue.getWaitingEvents + } + .filter(job => job != null && job.isInstanceOf[EntranceJob]) + .map(_.asInstanceOf[EntranceJob]) + .toArray + } + + /** + * to check timeout task,and kill timeout task timeout: default > 48h + */ + def startTimeOutCheck(): Unit = { + Utils.defaultScheduler.scheduleAtFixedRate( + new Runnable() { + override def run(): Unit = { + Utils.tryCatch { + + val timeoutType = EntranceConfiguration.ENTRANCE_TASK_TIMEOUT.getHotValue() + logger.info(s"Start to check timeout Job, timout is ${timeoutType}") + val timeoutTime = System.currentTimeMillis() - timeoutType.toLong + getAllUndoneTask(null, null).filter(job => job.createTime < timeoutTime).foreach { + job => + job.onFailure(s"Job has run for longer than the maximum time $timeoutType", null) + } + logger.info(s"Finished to check timeout Job, timout is ${timeoutType}") + } { case t: Throwable => + logger.warn(s"TimeoutDetective Job failed. ${t.getMessage}", t) + } + } + + }, + EntranceConfiguration.ENTRANCE_TASK_TIMEOUT_SCAN.getValue.toLong, + EntranceConfiguration.ENTRANCE_TASK_TIMEOUT_SCAN.getValue.toLong, + TimeUnit.MILLISECONDS + ) + } + + if (timeoutCheck) { + logger.info("Job time check is enabled") + startTimeOutCheck() + } + } object EntranceServer { diff --git a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/EntranceWebSocketService.scala b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/EntranceWebSocketService.scala index 714b9f0cc2..b5339c9e2e 100644 --- a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/EntranceWebSocketService.scala +++ b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/EntranceWebSocketService.scala @@ -215,18 +215,6 @@ class EntranceWebSocketService s"Your job's execution code is (after variable substitution and code check) " ) ) - entranceServer.getEntranceContext - .getOrCreateLogManager() - .onLogUpdate( - job, - "************************************SCRIPT CODE************************************" - ) - entranceServer.getEntranceContext - .getOrCreateLogManager() - .onLogUpdate( - job, - "************************************SCRIPT CODE************************************" - ) entranceServer.getEntranceContext .getOrCreateLogManager() .onLogUpdate( diff --git a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/conf/EntranceConfiguration.scala b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/conf/EntranceConfiguration.scala index e053d3793c..6f634bd76b 100644 --- a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/conf/EntranceConfiguration.scala +++ b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/conf/EntranceConfiguration.scala @@ -220,10 +220,32 @@ object EntranceConfiguration { val CREATOR_IP_SWITCH = CommonVars("wds.linkis.entrance.user.creator.ip.interceptor.switch", false) + val TEMPLATE_CONF_SWITCH = + CommonVars("linkis.entrance.template.conf.interceptor.switch", false) + val ENABLE_ENTRANCE_DIRTY_DATA_CLEAR = CommonVars("linkis.entrance.auto.clean.dirty.data.enable", false) - val ENTRANCE_FAILOVER_ENABLED = CommonVars("linkis.entrance.failover.enable", true).getValue + val ENTRANCE_CREATOR_JOB_LIMIT: CommonVars[Int] = + CommonVars[Int]( + "linkis.entrance.creator.job.concurrency.limit", + 10000, + "Creator task concurrency limit parameters" + ) + + val ENTRANCE_CREATOR_JOB_LIMIT_CONF_CACHE = + CommonVars("linkis.entrance.creator.job.concurrency.limit.conf.cache.time", 30L) + + val ENTRANCE_TASK_TIMEOUT = + CommonVars("linkis.entrance.task.timeout", new TimeType("48h")) + + val ENTRANCE_TASK_TIMEOUT_SCAN = + CommonVars("linkis.entrance.task.timeout.scan", new TimeType("12h")) + + val ENABLE_HDFS_JVM_USER = + CommonVars[Boolean]("linkis.entrance.enable.hdfs.jvm.user", true).getValue + + val ENTRANCE_FAILOVER_ENABLED = CommonVars("linkis.entrance.failover.enable", false).getValue val ENTRANCE_FAILOVER_SCAN_INIT_TIME = CommonVars("linkis.entrance.failover.scan.init.time", 3 * 1000).getValue @@ -247,7 +269,7 @@ object EntranceConfiguration { val ENTRANCE_SHUTDOWN_FAILOVER_CONSUME_QUEUE_ENABLED = CommonVars("linkis.entrance.shutdown.failover.consume.queue.enable", true).getValue - val ENTRANCE_GROUP_SCAN_ENABLED = CommonVars("linkis.entrance.group.scan.enable", true) + val ENTRANCE_GROUP_SCAN_ENABLED = CommonVars("linkis.entrance.group.scan.enable", false) val ENTRANCE_GROUP_SCAN_INIT_TIME = CommonVars("linkis.entrance.group.scan.init.time", 3 * 1000) diff --git a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/interceptor/impl/CommentInterceptor.scala b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/interceptor/impl/CommentInterceptor.scala index 627ab82b8e..bceb0f4f57 100644 --- a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/interceptor/impl/CommentInterceptor.scala +++ b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/interceptor/impl/CommentInterceptor.scala @@ -18,7 +18,6 @@ package org.apache.linkis.entrance.interceptor.impl import org.apache.linkis.common.utils.CodeAndRunTypeUtils -import org.apache.linkis.entrance.conf.EntranceConfiguration import org.apache.linkis.entrance.interceptor.EntranceInterceptor import org.apache.linkis.governance.common.entity.job.JobRequest import org.apache.linkis.manager.label.utils.LabelUtil @@ -70,8 +69,38 @@ trait CommentHelper { object SQLCommentHelper extends CommentHelper { override val commentPattern: Regex = """\s*--.+\s*""".r.unanchored private val comment = "(?ms)('(?:''|[^'])*')|--.*?$|/\\*.*?\\*/|#.*?$|" + private val comment_sem = "(?i)(comment)\\s+'([^']*)'" private val logger: Logger = LoggerFactory.getLogger(getClass) + def replaceComment(code: String): String = { + try { + val pattern = Pattern.compile(comment_sem) + val matcher = pattern.matcher(code) + val sb = new StringBuffer + while (matcher.find()) { + val commentKeyword = matcher.group(1) + val comment = matcher.group(2) + + /** + * Since we are in a Scala string, and each backslash needs to be escaped in the string + * itself, we need two additional backslashes. Therefore, we end up with a total of four + * backslashes to represent a single literal backslash in the replacement string. + */ + val escapedComment = comment.replaceAll(";", "\\\\\\\\;") + matcher.appendReplacement(sb, commentKeyword + " '" + escapedComment + "'") + } + matcher.appendTail(sb) + sb.toString + } catch { + case e: Exception => + logger.warn("sql comment semicolon replace failed") + code + case t: Throwable => + logger.warn("sql comment semicolon replace failed") + code + } + } + override def dealComment(code: String): String = { try { val p = Pattern.compile(comment) diff --git a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/interceptor/impl/Explain.scala b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/interceptor/impl/Explain.scala index 8436ccc711..045cb51f88 100644 --- a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/interceptor/impl/Explain.scala +++ b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/interceptor/impl/Explain.scala @@ -57,6 +57,8 @@ object SparkExplain extends Explain { private val sy = Pattern.compile("sys\\.") private val scCancelAllJobs = Pattern.compile("sc\\.cancelAllJobs(\\s*)") private val runtime = Pattern.compile("Runtime\\.getRuntime") + private val LINE_BREAK = "\n" + private val LOG: Logger = LoggerFactory.getLogger(getClass) override def authPass(code: String, error: StringBuilder): Boolean = { if (EntranceConfiguration.SKIP_AUTH.getHotValue()) { @@ -99,6 +101,7 @@ object SQLExplain extends Explain { private val LIMIT: String = "limit" private val LIMIT_UPPERCASE: String = "LIMIT" private val IDE_ALLOW_NO_LIMIT = "--set wds.linkis.engine.no.limit.allow=true" + private val LOG: Logger = LoggerFactory.getLogger(getClass) override def authPass(code: String, error: StringBuilder): Boolean = { true @@ -118,7 +121,8 @@ object SQLExplain extends Explain { logAppender: java.lang.StringBuilder ): Unit = { val fixedCode: ArrayBuffer[String] = new ArrayBuffer[String]() - val tempCode = SQLCommentHelper.dealComment(executionCode) + val tempCode1 = SQLCommentHelper.dealComment(executionCode) + val tempCode = SQLCommentHelper.replaceComment(tempCode1) val isNoLimitAllowed = Utils.tryCatch { IDE_ALLOW_NO_LIMIT_REGEX.findFirstIn(executionCode).isDefined } { case e: Exception => @@ -131,6 +135,8 @@ object SQLExplain extends Explain { .generateWarn("please pay attention ,SQL full export mode opened(请注意,SQL全量导出模式打开)\n") ) } + var isFirstTimePrintingLimit = true + var isFirstTimePrintingOverLimit = true if (tempCode.contains("""\;""")) { val semicolonIndexes = findRealSemicolonIndex(tempCode) var oldIndex = 0 @@ -140,20 +146,27 @@ object SQLExplain extends Explain { if (isSelectCmd(singleCode)) { val trimCode = singleCode.trim if (isSelectCmdNoLimit(trimCode) && !isNoLimitAllowed) { - logAppender.append( - LogUtils.generateWarn( - s"You submitted a sql without limit, DSS will add limit 5000 to your sql" - ) + "\n" - ) + if (isFirstTimePrintingLimit) { + logAppender.append( + LogUtils.generateWarn( + s"You submitted a sql without limit, DSS will add limit 5000 to your sql" + ) + "\n" + ) + isFirstTimePrintingLimit = false + } + // 将注释先干掉,然后再进行添加limit val realCode = cleanComment(trimCode) fixedCode += (realCode + SQL_APPEND_LIMIT) } else if (isSelectOverLimit(singleCode) && !isNoLimitAllowed) { val trimCode = singleCode.trim - logAppender.append( - LogUtils.generateWarn( - s"You submitted a sql with limit exceeding 5000, it is not allowed. DSS will change your limit to 5000" - ) + "\n" - ) + if (isFirstTimePrintingOverLimit) { + logAppender.append( + LogUtils.generateWarn( + s"You submitted a sql with limit exceeding 5000, it is not allowed. DSS will change your limit to 5000" + ) + "\n" + ) + isFirstTimePrintingOverLimit = false + } fixedCode += repairSelectOverLimit(trimCode) } else { fixedCode += singleCode.trim @@ -167,20 +180,27 @@ object SQLExplain extends Explain { if (isSelectCmd(singleCode)) { val trimCode = singleCode.trim if (isSelectCmdNoLimit(trimCode) && !isNoLimitAllowed) { - logAppender.append( - LogUtils.generateWarn( - s"You submitted a sql without limit, DSS will add limit 5000 to your sql" - ) + "\n" - ) + if (isFirstTimePrintingLimit) { + logAppender.append( + LogUtils.generateWarn( + s"You submitted a sql without limit, DSS will add limit 5000 to your sql" + ) + "\n" + ) + isFirstTimePrintingLimit = false + } + // 将注释先干掉,然后再进行添加limit val realCode = cleanComment(trimCode) fixedCode += (realCode + SQL_APPEND_LIMIT) } else if (isSelectOverLimit(singleCode) && !isNoLimitAllowed) { val trimCode = singleCode.trim - logAppender.append( - LogUtils.generateWarn( - s"You submitted a sql with limit exceeding 5000, it is not allowed. DSS will change your limit to 5000" - ) + "\n" - ) + if (isFirstTimePrintingOverLimit) { + logAppender.append( + LogUtils.generateWarn( + s"You submitted a sql with limit exceeding 5000, it is not allowed. DSS will change your limit to 5000" + ) + "\n" + ) + isFirstTimePrintingOverLimit = false + } fixedCode += repairSelectOverLimit(trimCode) } else { fixedCode += singleCode.trim @@ -210,6 +230,8 @@ object SQLExplain extends Explain { array.toArray } + private def addNoLimit(code: String) = code + NO_LIMIT_STRING + protected def needNoLimit(code: String): Boolean = code.endsWith(NO_LIMIT_STRING) def isSelectCmd(code: String): Boolean = { @@ -217,16 +239,17 @@ object SQLExplain extends Explain { return false } val realCode = cleanComment(code) - realCode.trim.split("\\s+")(0).toLowerCase().contains("select") + realCode.trim.split("\\s+")(0).toLowerCase(Locale.getDefault).contains("select") } - def continueWhenError: Boolean = false + // def continueWhenError = false def isSelectCmdNoLimit(cmd: String): Boolean = { if (StringUtils.isEmpty(cmd)) { return false } val realCode = cmd.trim + // limit is often the last in a sql statement, so you need to make a final judgment val arr = realCode.split("\\s+") val words = new ArrayBuffer[String]() arr foreach { w => @@ -235,8 +258,10 @@ object SQLExplain extends Explain { val a = words.toArray val length = a.length if (a.length > 1) { - val second_last = a(length - 2) - !"limit".equals(second_last.toLowerCase()) + val second_last = a(length - 2).toLowerCase(Locale.getDefault) + // for some case eg:"SELECT * from dual WHERE (1=1)LIMIT 1;" + val result = !("limit".equals(second_last) || second_last.contains(")limit")) + result } else { false } diff --git a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/interceptor/impl/TemplateConfInterceptor.scala b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/interceptor/impl/TemplateConfInterceptor.scala new file mode 100644 index 0000000000..6accd30bd5 --- /dev/null +++ b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/interceptor/impl/TemplateConfInterceptor.scala @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.entrance.interceptor.impl + +import org.apache.linkis.entrance.conf.EntranceConfiguration +import org.apache.linkis.entrance.interceptor.EntranceInterceptor +import org.apache.linkis.governance.common.entity.job.JobRequest + +import java.lang + +class TemplateConfInterceptor extends EntranceInterceptor { + + override def apply(jobRequest: JobRequest, logAppender: lang.StringBuilder): JobRequest = { + if (EntranceConfiguration.TEMPLATE_CONF_SWITCH.getValue) { + TemplateConfUtils.dealWithTemplateConf(jobRequest, logAppender) + } else { + jobRequest + } + } + +} diff --git a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/interceptor/impl/TemplateConfUtils.scala b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/interceptor/impl/TemplateConfUtils.scala new file mode 100644 index 0000000000..cdcbe01e85 --- /dev/null +++ b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/interceptor/impl/TemplateConfUtils.scala @@ -0,0 +1,276 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.entrance.interceptor.impl + +import org.apache.linkis.common.conf.Configuration +import org.apache.linkis.common.exception.LinkisCommonErrorException +import org.apache.linkis.common.log.LogUtils +import org.apache.linkis.common.utils.{CodeAndRunTypeUtils, Logging, Utils} +import org.apache.linkis.entrance.conf.EntranceConfiguration +import org.apache.linkis.governance.common.entity.TemplateConfKey +import org.apache.linkis.governance.common.entity.job.JobRequest +import org.apache.linkis.governance.common.protocol.conf.{TemplateConfRequest, TemplateConfResponse} +import org.apache.linkis.manager.label.builder.factory.LabelBuilderFactoryContext +import org.apache.linkis.manager.label.constant.LabelKeyConstant +import org.apache.linkis.manager.label.entity.entrance.ExecuteOnceLabel +import org.apache.linkis.manager.label.utils.LabelUtil +import org.apache.linkis.protocol.utils.TaskUtils +import org.apache.linkis.rpc.Sender + +import org.apache.commons.lang3.StringUtils + +import java.{lang, util} +import java.util.concurrent.TimeUnit + +import scala.collection.JavaConverters._ + +import com.google.common.cache.{CacheBuilder, CacheLoader, LoadingCache} + +object TemplateConfUtils extends Logging { + + val confTemplateNameKey = "ec.resource.name" + + private val templateCache: LoadingCache[String, util.List[TemplateConfKey]] = CacheBuilder + .newBuilder() + .maximumSize(1000) + .expireAfterWrite(5, TimeUnit.MINUTES) + .build(new CacheLoader[String, util.List[TemplateConfKey]]() { + + override def load(templateUuid: String): util.List[TemplateConfKey] = { + var templateList = Utils.tryAndWarn { + val sender: Sender = Sender + .getSender(Configuration.CLOUD_CONSOLE_CONFIGURATION_SPRING_APPLICATION_NAME.getValue) + + logger.info(s"load template configuration data templateUuid:$templateUuid") + val res = sender.ask(new TemplateConfRequest(templateUuid)) match { + case response: TemplateConfResponse => + logger + .debug(s"${response.getList()}") + response.getList + case _ => + logger + .warn(s"load template configuration data templateUuid:$templateUuid loading failed") + new util.ArrayList[TemplateConfKey](0) + } + res + } + if (templateList.size() == 0) { + logger.warn(s"template configuration data loading failed, plaese check warn log") + } + templateList + } + + }) + + private val templateCacheName: LoadingCache[String, util.List[TemplateConfKey]] = CacheBuilder + .newBuilder() + .maximumSize(1000) + .expireAfterWrite(5, TimeUnit.MINUTES) + .build(new CacheLoader[String, util.List[TemplateConfKey]]() { + + override def load(templateName: String): util.List[TemplateConfKey] = { + var templateList = Utils.tryAndWarn { + val sender: Sender = Sender + .getSender(Configuration.CLOUD_CONSOLE_CONFIGURATION_SPRING_APPLICATION_NAME.getValue) + + logger.info(s"load template configuration data templateName:$templateName") + val res = sender.ask(new TemplateConfRequest(null, templateName)) match { + case response: TemplateConfResponse => + logger + .debug(s"${response.getList()}") + response.getList + case _ => + logger + .warn(s"load template configuration data templateName:$templateName loading failed") + new util.ArrayList[TemplateConfKey](0) + } + res + } + + if (templateList.size() == 0) { + logger.warn(s"template configuration data loading failed, plaese check warn log") + } + templateList + } + + }) + + /** + * Get user-defined template conf name value + * + * @param code + * :code + * @param codeType + * :sql,hql,scala + * @return + * String the last one of template conf name + */ + def getCustomTemplateConfName(code: String, codeType: String): String = { + var templateConfName = ""; + + var varString: String = null + var errString: String = null + var rightVarString: String = null + + val languageType = CodeAndRunTypeUtils.getLanguageTypeByCodeType(codeType) + + languageType match { + case CodeAndRunTypeUtils.LANGUAGE_TYPE_SQL => + varString = s"""\\s*---@set ${confTemplateNameKey}=\\s*.+\\s*""" + errString = """\s*---@.*""" + case CodeAndRunTypeUtils.LANGUAGE_TYPE_PYTHON | CodeAndRunTypeUtils.LANGUAGE_TYPE_SHELL => + varString = s"""\\s*##@set ${confTemplateNameKey}=\\s*.+\\s*""" + errString = """\s*##@""" + case CodeAndRunTypeUtils.LANGUAGE_TYPE_SCALA => + varString = s"""\\s*///@set ${confTemplateNameKey}=\\s*.+\\s*""" + errString = """\s*///@.+""" + case _ => + return templateConfName + } + + val customRegex = varString.r.unanchored + val errRegex = errString.r.unanchored + var codeRes = code.replaceAll("\r\n", "\n") + // only allow set at fisrt line + val res = codeRes.split("\n") + if (res.size > 0) { + val str = res(0) + str match { + case customRegex() => + val clearStr = if (str.endsWith(";")) str.substring(0, str.length - 1) else str + val res: Array[String] = clearStr.split("=") + if (res != null && res.length == 2) { + templateConfName = res(1).trim + logger.info(s"get template conf name $templateConfName") + } else { + if (res.length > 2) { + throw new LinkisCommonErrorException( + 20044, + s"$str template conf name var defined uncorrectly" + ) + } else { + throw new LinkisCommonErrorException( + 20045, + s"template conf name var was defined uncorrectly:$str" + ) + } + } + case errRegex() => + logger.warn( + s"The template conf name var definition is incorrect:$str,if it is not used, it will not run the error, but it is recommended to use the correct specification to define" + ) + case _ => + } + } + templateConfName + } + + def dealWithTemplateConf(jobRequest: JobRequest, logAppender: lang.StringBuilder): JobRequest = { + jobRequest match { + case requestPersistTask: JobRequest => + val params = requestPersistTask.getParams + val startMap = TaskUtils.getStartupMap(params) + + var templateConflist: util.List[TemplateConfKey] = new util.ArrayList[TemplateConfKey]() + var templateName: String = "" + // only for Creator:IDE, try to get template conf name from code string. eg:---@set ec.resource.name=xxxx + val (user, creator) = LabelUtil.getUserCreator(jobRequest.getLabels) + if (EntranceConfiguration.DEFAULT_REQUEST_APPLICATION_NAME.getValue.equals(creator)) { + val codeType = LabelUtil.getCodeType(jobRequest.getLabels) + templateName = + TemplateConfUtils.getCustomTemplateConfName(jobRequest.getExecutionCode, codeType) + } + + // code template name > start params template uuid + if (StringUtils.isBlank(templateName)) { + logger.debug("jobRequest startMap param template name is empty") + + logger.info("jobRequest startMap params :{} ", startMap) + val templateUuid = startMap.getOrDefault(LabelKeyConstant.TEMPLATE_CONF_KEY, "").toString + + if (StringUtils.isBlank(templateUuid)) { + logger.debug("jobRequest startMap param template id is empty") + } else { + logger.info("try to get template conf list with template uid:{} ", templateUuid) + logAppender.append( + LogUtils + .generateInfo(s"Try to get template conf data with template uid:$templateUuid\nn") + ) + templateConflist = templateCache.get(templateUuid) + if (templateConflist == null || templateConflist.size() == 0) { + logAppender.append( + LogUtils.generateWarn( + s"Can not get any template conf data with template uid:$templateUuid\n" + ) + ) + } + } + } else { + logger.info("Try to get template conf list with template name:[{}]", templateName) + logAppender.append( + LogUtils + .generateInfo(s"Try to get template conf data with template name:[$templateName]\n") + ) + templateConflist = templateCacheName.get(templateName) + if (templateConflist == null || templateConflist.size() == 0) { + logAppender.append( + LogUtils.generateWarn( + s"Can not get any template conf data with template name:$templateName\n" + ) + ) + } else { + // to remove metedata start param + TaskUtils.clearStartupMap(params) + + val onceLabel = + LabelBuilderFactoryContext.getLabelBuilderFactory.createLabel( + classOf[ExecuteOnceLabel] + ) + logger.info("Add once label for task id:{}", requestPersistTask.getId.toString) + requestPersistTask.getLabels.add(onceLabel) + } + } + + if (templateConflist != null && templateConflist.size() > 0) { + val keyList = new util.HashMap[String, AnyRef]() + templateConflist.asScala.foreach(ele => { + val key = ele.getKey + val oldValue = startMap.get(key) + if (oldValue != null && StringUtils.isNotBlank(oldValue.toString)) { + logger.info(s"key:$key value:$oldValue not empty, skip to deal") + } else { + val newValue = ele.getConfigValue + logger.info(s"key:$key value:$newValue will add to startMap params") + if (TaskUtils.isWithDebugInfo(params)) { + logAppender.append(LogUtils.generateInfo(s"add $key=$newValue\n")) + } + keyList.put(key, newValue) + } + + }) + if (keyList.size() > 0) { + TaskUtils.addStartupMap(params, keyList) + } + } + + case _ => + } + jobRequest + } + +} diff --git a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/interceptor/impl/VarSubstitutionInterceptor.scala b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/interceptor/impl/VarSubstitutionInterceptor.scala index 0487a238cf..72d40305a6 100644 --- a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/interceptor/impl/VarSubstitutionInterceptor.scala +++ b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/interceptor/impl/VarSubstitutionInterceptor.scala @@ -41,10 +41,26 @@ class VarSubstitutionInterceptor extends EntranceInterceptor { LogUtils.generateInfo("Program is substituting variables for you") + "\n" ) val codeType = LabelUtil.getCodeType(jobRequest.getLabels) - jobRequest.setExecutionCode(CustomVariableUtils.replaceCustomVar(jobRequest, codeType)) + val realCode = CustomVariableUtils.replaceCustomVar(jobRequest, codeType) + jobRequest.setExecutionCode(realCode) logAppender.append( LogUtils.generateInfo("Variables substitution ended successfully") + "\n" ) + // print code after variables substitution + logAppender.append( + LogUtils.generateInfo( + "You have submitted a new job, script code (after variable substitution) is" + ) + "\n" + ); + logAppender.append( + "************************************SCRIPT CODE************************************" + "\n" + ) + logAppender.append(realCode); + logAppender.append("\n"); + logAppender.append( + "************************************SCRIPT CODE************************************" + "\n" + ); + jobRequest } { case e: VarSubstitutionException => diff --git a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/log/CacheLogReader.scala b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/log/CacheLogReader.scala index 483cf9ab43..748f82df4b 100644 --- a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/log/CacheLogReader.scala +++ b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/log/CacheLogReader.scala @@ -19,7 +19,11 @@ package org.apache.linkis.entrance.log import org.apache.linkis.common.io.{Fs, FsPath} import org.apache.linkis.common.utils.Utils +import org.apache.linkis.entrance.conf.EntranceConfiguration +import org.apache.linkis.entrance.exception.LogReadFailedException import org.apache.linkis.storage.FSFactory +import org.apache.linkis.storage.fs.FileSystem +import org.apache.linkis.storage.utils.StorageUtils import java.io.{InputStream, IOException} import java.util @@ -36,13 +40,26 @@ class CacheLogReader(logPath: String, charset: String, sharedCache: Cache, user: var closed = false private def createInputStream: InputStream = { + if (!logPath.contains(user)) { + throw new LogReadFailedException( + s"${user} does not have permission to read the path $logPath" + ) + } + val fsPath = new FsPath(logPath) if (fileSystem == null) lock synchronized { if (fileSystem == null) { - fileSystem = FSFactory.getFsByProxyUser(new FsPath(logPath), user) + + fileSystem = + if (StorageUtils.isHDFSPath(fsPath) && EntranceConfiguration.ENABLE_HDFS_JVM_USER) { + FSFactory.getFs(new FsPath(logPath)).asInstanceOf[FileSystem] + } else { + FSFactory.getFsByProxyUser(new FsPath(logPath), user).asInstanceOf[FileSystem] + } + fileSystem.init(new util.HashMap[String, String]()) } } - val inputStream: InputStream = fileSystem.read(new FsPath(logPath)) + val inputStream: InputStream = fileSystem.read(fsPath) inputStream } diff --git a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/log/CacheLogWriter.scala b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/log/CacheLogWriter.scala index 9028c469ab..b54dc757cd 100644 --- a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/log/CacheLogWriter.scala +++ b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/log/CacheLogWriter.scala @@ -41,6 +41,8 @@ class CacheLogWriter(logPath: String, charset: String, sharedCache: Cache, user: val sb = new StringBuilder if (removed != null) sb.append(removed).append("\n") logs.filter(_ != null).foreach(log => sb.append(log).append("\n")) + // need append latest msg before clear + sb.append(msg).append("\n") sharedCache.cachedLogs.fakeClear() super.write(sb.toString()) pushTime.setTime( diff --git a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/log/ErrorCodeManager.scala b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/log/ErrorCodeManager.scala index 54914b6002..4b082342ce 100644 --- a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/log/ErrorCodeManager.scala +++ b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/log/ErrorCodeManager.scala @@ -18,6 +18,12 @@ package org.apache.linkis.entrance.log import org.apache.linkis.errorcode.client.handler.LinkisErrorCodeHandler +import org.apache.linkis.errorcode.client.manager.LinkisErrorCodeManager +import org.apache.linkis.errorcode.common.LinkisErrorCode + +import java.util + +import scala.collection.JavaConverters.asScalaBufferConverter abstract class ErrorCodeManager { @@ -35,6 +41,22 @@ abstract class ErrorCodeManager { None } + def errorMatchAndGetContent(log: String): Option[(String, String, String)] = { + getErrorCodes.foreach(e => + if (e.regex.findFirstIn(log).isDefined) { + val matched = e.regex.unapplySeq(log) + if (matched.nonEmpty) { + return Some( + e.code, + e.message.format(matched.get: _*), + e.regex.findFirstIn(log).getOrElse("") + ) + } else Some(e.code, e.message, "") + } + ) + None + } + } /** @@ -44,7 +66,24 @@ object FlexibleErrorCodeManager extends ErrorCodeManager { private val errorCodeHandler = LinkisErrorCodeHandler.getInstance() - override def getErrorCodes: Array[ErrorCode] = Array.empty + private val linkisErrorCodeManager = LinkisErrorCodeManager.getInstance + + override def getErrorCodes: Array[ErrorCode] = { + val errorCodes: util.List[LinkisErrorCode] = linkisErrorCodeManager.getLinkisErrorCodes + if (errorCodes == null) { + Array.empty + } else { + errorCodes.asScala + .map(linkisErrorCode => + ErrorCode( + linkisErrorCode.getErrorRegex, + linkisErrorCode.getErrorCode, + linkisErrorCode.getErrorDesc + ) + ) + .toArray + } + } override def errorMatch(log: String): Option[(String, String)] = { val errorCodes = errorCodeHandler.handle(log) diff --git a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/log/HDFSCacheLogWriter.scala b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/log/HDFSCacheLogWriter.scala index 24633dfbb2..4f37ff1040 100644 --- a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/log/HDFSCacheLogWriter.scala +++ b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/log/HDFSCacheLogWriter.scala @@ -37,11 +37,15 @@ import java.util class HDFSCacheLogWriter(logPath: String, charset: String, sharedCache: Cache, user: String) extends LogWriter(charset) { - if (StringUtils.isBlank(logPath)) + if (StringUtils.isBlank(logPath)) { throw new EntranceErrorException(LOGPATH_NOT_NULL.getErrorCode, LOGPATH_NOT_NULL.getErrorDesc) + } - protected var fileSystem = + protected var fileSystem = if (EntranceConfiguration.ENABLE_HDFS_JVM_USER) { + FSFactory.getFs(new FsPath(logPath)).asInstanceOf[FileSystem] + } else { FSFactory.getFsByProxyUser(new FsPath(logPath), user).asInstanceOf[FileSystem] + } override protected var outputStream: OutputStream = null @@ -55,7 +59,12 @@ class HDFSCacheLogWriter(logPath: String, charset: String, sharedCache: Cache, u private def init(): Unit = { fileSystem.init(new util.HashMap[String, String]()) - FileSystemUtils.createNewFileWithFileSystem(fileSystem, new FsPath(logPath), user, true) + FileSystemUtils.createNewFileAndSetOwnerWithFileSystem( + fileSystem, + new FsPath(logPath), + user, + true + ) } @throws[IOException] @@ -99,6 +108,8 @@ class HDFSCacheLogWriter(logPath: String, charset: String, sharedCache: Cache, u val sb = new StringBuilder if (removed != null) sb.append(removed).append("\n") logs.filter(_ != null).foreach(log => sb.append(log).append("\n")) + // need append latest msg before fake clear + sb.append(msg).append("\n") sharedCache.cachedLogs.fakeClear() writeToFile(sb.toString()) pushTime.setTime( diff --git a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/log/LogManager.scala b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/log/LogManager.scala index 626a643a0b..19f4c5c6ad 100644 --- a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/log/LogManager.scala +++ b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/log/LogManager.scala @@ -17,6 +17,7 @@ package org.apache.linkis.entrance.log +import org.apache.linkis.common.log.LogUtils import org.apache.linkis.common.utils.{Logging, Utils} import org.apache.linkis.entrance.EntranceContext import org.apache.linkis.entrance.job.EntranceExecutionJob @@ -59,10 +60,19 @@ abstract class LogManager extends LogListener with Logging { } } } - entranceExecutionJob.getLogWriter.foreach(logWriter => logWriter.write(log)) - errorCodeManager.foreach(_.errorMatch(log).foreach { case (code, errorMsg) => - errorCodeListener.foreach(_.onErrorCodeCreated(job, code, errorMsg)) + var writeLog = log + errorCodeManager.foreach(_.errorMatchAndGetContent(log).foreach { + case (code, errorMsg, targetMsg) => + if (!targetMsg.contains(LogUtils.ERROR_STR) && log.contains(LogUtils.ERROR_STR)) { + writeLog = LogUtils.generateERROR( + s"error code: $code, errorMsg: $errorMsg, errorLine: $targetMsg \n" + log + ) + } + errorCodeListener.foreach(_.onErrorCodeCreated(job, code, errorMsg)) + case _ => }) + entranceExecutionJob.getLogWriter.foreach(logWriter => logWriter.write(writeLog)) + case _ => } } { diff --git a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/log/LoopArray.scala b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/log/LoopArray.scala index 155d8c7bd5..5b62a49aa1 100644 --- a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/log/LoopArray.scala +++ b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/log/LoopArray.scala @@ -51,7 +51,7 @@ class LoopArray[T](maxCapacity: Int) { } else if (index > _max) { throw new IllegalArgumentException("The index " + index + " must be less than " + _max) } - val _index = (flag + (index - realSize)) % maxCapacity + val _index = (flag + (index - realSize + maxCapacity - 1)) % maxCapacity eventQueue(_index).asInstanceOf[T] } diff --git a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/orchestrator/plugin/EntranceUserParallelOrchestratorPlugin.scala b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/orchestrator/plugin/EntranceUserParallelOrchestratorPlugin.scala index 4b9b4570f1..e5c657023e 100644 --- a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/orchestrator/plugin/EntranceUserParallelOrchestratorPlugin.scala +++ b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/orchestrator/plugin/EntranceUserParallelOrchestratorPlugin.scala @@ -20,19 +20,15 @@ package org.apache.linkis.entrance.orchestrator.plugin import org.apache.linkis.common.conf.Configuration import org.apache.linkis.common.utils.{Logging, Utils} import org.apache.linkis.entrance.conf.EntranceConfiguration +import org.apache.linkis.entrance.utils.EntranceUtils import org.apache.linkis.governance.common.protocol.conf.{ RequestQueryEngineConfigWithGlobalConfig, ResponseQueryConfig } -import org.apache.linkis.manager.label.builder.factory.LabelBuilderFactoryContext -import org.apache.linkis.manager.label.constant.LabelKeyConstant import org.apache.linkis.manager.label.entity.Label import org.apache.linkis.manager.label.entity.engine.{EngineTypeLabel, UserCreatorLabel} import org.apache.linkis.orchestrator.plugin.UserParallelOrchestratorPlugin import org.apache.linkis.rpc.Sender -import org.apache.linkis.server.BDPJettyServerHelper - -import org.apache.commons.lang3.StringUtils import java.util import java.util.concurrent.TimeUnit @@ -43,10 +39,6 @@ import com.google.common.cache.{CacheBuilder, CacheLoader, LoadingCache} class EntranceUserParallelOrchestratorPlugin extends UserParallelOrchestratorPlugin with Logging { - private val SPLIT = "," - - private val labelFactory = LabelBuilderFactoryContext.getLabelBuilderFactory - private def getDefaultMaxRuningNum: Int = { EntranceConfiguration.WDS_LINKIS_INSTANCE.getHotValue() } @@ -62,7 +54,7 @@ class EntranceUserParallelOrchestratorPlugin extends UserParallelOrchestratorPlu .build(new CacheLoader[String, Integer]() { override def load(key: String): Integer = { - val (userCreatorLabel, engineTypeLabel) = fromKeyGetLabels(key) + val (userCreatorLabel, engineTypeLabel) = EntranceUtils.fromKeyGetLabels(key) val keyAndValue = Utils.tryAndWarnMsg { sender .ask(RequestQueryEngineConfigWithGlobalConfig(userCreatorLabel, engineTypeLabel)) @@ -75,10 +67,8 @@ class EntranceUserParallelOrchestratorPlugin extends UserParallelOrchestratorPlu null == keyAndValue || !keyAndValue .containsKey(EntranceConfiguration.WDS_LINKIS_INSTANCE.key) ) { - logger.error( - s"cannot found user configuration key:${EntranceConfiguration.WDS_LINKIS_INSTANCE.key}," + - s"will use default value ${EntranceConfiguration.WDS_LINKIS_INSTANCE.getHotValue()}。All config map: ${BDPJettyServerHelper.gson - .toJson(keyAndValue)}" + logger.warn( + s"cannot found user configuration key:${EntranceConfiguration.WDS_LINKIS_INSTANCE.key}," + s"will use default value " ) } val maxRunningJobs = EntranceConfiguration.WDS_LINKIS_INSTANCE.getValue(keyAndValue, true) @@ -102,27 +92,7 @@ class EntranceUserParallelOrchestratorPlugin extends UserParallelOrchestratorPlu if (null == userCreatorLabel || null == engineTypeLabel) { return getDefaultMaxRuningNum } - configCache.get(getKey(userCreatorLabel, engineTypeLabel)) - } - - private def getKey( - userCreatorLabel: UserCreatorLabel, - engineTypeLabel: EngineTypeLabel - ): String = { - userCreatorLabel.getStringValue + SPLIT + engineTypeLabel.getStringValue - } - - private def fromKeyGetLabels(key: String): (UserCreatorLabel, EngineTypeLabel) = { - if (StringUtils.isBlank(key)) (null, null) - else { - val labelStringValues = key.split(SPLIT) - if (labelStringValues.length < 2) return (null, null) - val userCreatorLabel = labelFactory - .createLabel[UserCreatorLabel](LabelKeyConstant.USER_CREATOR_TYPE_KEY, labelStringValues(0)) - val engineTypeLabel = labelFactory - .createLabel[EngineTypeLabel](LabelKeyConstant.ENGINE_TYPE_KEY, labelStringValues(1)) - (userCreatorLabel, engineTypeLabel) - } + configCache.get(EntranceUtils.getUserCreatorEcTypeKey(userCreatorLabel, engineTypeLabel)) } override def isReady: Boolean = true diff --git a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/persistence/EntranceResultSetEngine.scala b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/persistence/EntranceResultSetEngine.scala index 02d1a6a08e..2ba98438e8 100644 --- a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/persistence/EntranceResultSetEngine.scala +++ b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/persistence/EntranceResultSetEngine.scala @@ -17,20 +17,12 @@ package org.apache.linkis.entrance.persistence -import org.apache.linkis.common.io.{FsPath, MetaData, Record} -import org.apache.linkis.common.io.resultset.ResultSet -import org.apache.linkis.common.utils.{Logging, Utils} +import org.apache.linkis.common.utils.Logging import org.apache.linkis.entrance.exception.{EntranceErrorCode, EntranceErrorException} -import org.apache.linkis.entrance.execute.StorePathExecuteRequest -import org.apache.linkis.entrance.job.{EntranceExecuteRequest, EntranceExecutionJob} -import org.apache.linkis.entrance.scheduler.cache.CacheOutputExecuteResponse -import org.apache.linkis.governance.common.entity.job.SubJobDetail import org.apache.linkis.scheduler.executer.{AliasOutputExecuteResponse, OutputExecuteResponse} import org.apache.linkis.scheduler.queue.Job -import org.apache.linkis.storage.resultset.{ResultSetFactory, ResultSetWriterFactory} -import org.apache.linkis.storage.utils.FileSystemUtils +import org.apache.linkis.storage.resultset.ResultSetFactory -import org.apache.commons.io.IOUtils import org.apache.commons.lang3.StringUtils class EntranceResultSetEngine extends ResultSetEngine with Logging { @@ -46,15 +38,11 @@ class EntranceResultSetEngine extends ResultSetEngine with Logging { EntranceErrorCode.RESULT_NOT_PERSISTED_ERROR.getDesc ) } - case CacheOutputExecuteResponse(alias, output) => - if (ResultSetFactory.getInstance.isResultSetPath(output)) { - getDir(output) - } else { - throw new EntranceErrorException( - EntranceErrorCode.RESULT_NOT_PERSISTED_ERROR.getErrCode, - EntranceErrorCode.RESULT_NOT_PERSISTED_ERROR.getDesc - ) - } + case _ => + throw new EntranceErrorException( + EntranceErrorCode.RESULT_NOT_PERSISTED_ERROR.getErrCode, + EntranceErrorCode.RESULT_NOT_PERSISTED_ERROR.getDesc + ) } } @@ -64,7 +52,7 @@ class EntranceResultSetEngine extends ResultSetEngine with Logging { } else { val arr = str.split("/").filter(StringUtils.isNotBlank) if (arr.length <= 2) { - return str + str } else { str.substring(0, str.lastIndexOf("/")) } diff --git a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/scheduler/EntranceFIFOUserConsumer.scala b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/scheduler/EntranceFIFOUserConsumer.scala index e2f0ab1d5a..26d8a60c4c 100644 --- a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/scheduler/EntranceFIFOUserConsumer.scala +++ b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/scheduler/EntranceFIFOUserConsumer.scala @@ -17,7 +17,7 @@ package org.apache.linkis.entrance.scheduler -import org.apache.linkis.common.utils.Utils +import org.apache.linkis.common.utils.{Logging, Utils} import org.apache.linkis.entrance.conf.EntranceConfiguration import org.apache.linkis.entrance.job.EntranceExecutionJob import org.apache.linkis.entrance.utils.JobHistoryHelper @@ -34,7 +34,8 @@ class EntranceFIFOUserConsumer( schedulerContext: SchedulerContext, executeService: ExecutorService, private var group: Group -) extends FIFOUserConsumer(schedulerContext, executeService, group) { +) extends FIFOUserConsumer(schedulerContext, executeService, group) + with Logging { override def loop(): Unit = { // When offlineFlag=true, the unsubmitted tasks will be failover, and the running tasks will wait for completion. @@ -67,4 +68,39 @@ class EntranceFIFOUserConsumer( } + override def runScheduleIntercept: Boolean = { + val consumers = getSchedulerContext.getOrCreateConsumerManager.listConsumers + var creatorRunningJobNum = 0 + // APP_TEST_hadoop_hive or IDE_hadoop_hive + val groupNameStr = getGroup.getGroupName + val groupNames = groupNameStr.split("_") + val length = groupNames.length + if (length < 3) return true + // APP_TEST + val lastIndex = groupNameStr.lastIndexOf("_") + val secondLastIndex = groupNameStr.lastIndexOf("_", lastIndex - 1) + val creatorName = groupNameStr.substring(0, secondLastIndex) + // hive + val ecType = groupNames(length - 1) + for (consumer <- consumers) { + val groupName = consumer.getGroup.getGroupName + if (groupName.startsWith(creatorName) && groupName.endsWith(ecType)) { + creatorRunningJobNum += consumer.getRunningEvents.length + } + } + val creatorECTypeMaxRunningJobs = + CreatorECTypeDefaultConf.getCreatorECTypeMaxRunningJobs(creatorName, ecType) + if (logger.isDebugEnabled) { + logger.debug( + s"Creator: $creatorName EC:$ecType there are currently:$creatorRunningJobNum jobs running and maximum limit: $creatorECTypeMaxRunningJobs" + ) + } + if (creatorRunningJobNum > creatorECTypeMaxRunningJobs) { + logger.error( + s"Creator: $creatorName EC:$ecType there are currently:$creatorRunningJobNum jobs running that exceed the maximum limit: $creatorECTypeMaxRunningJobs" + ) + false + } else true + } + } diff --git a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/scheduler/EntranceGroupFactory.scala b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/scheduler/EntranceGroupFactory.scala index 0f31351b48..de4c025e30 100644 --- a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/scheduler/EntranceGroupFactory.scala +++ b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/scheduler/EntranceGroupFactory.scala @@ -23,19 +23,14 @@ import org.apache.linkis.entrance.conf.EntranceConfiguration import org.apache.linkis.entrance.errorcode.EntranceErrorCodeSummary._ import org.apache.linkis.entrance.exception.{EntranceErrorCode, EntranceErrorException} import org.apache.linkis.entrance.execute.EntranceJob +import org.apache.linkis.entrance.utils.EntranceUtils import org.apache.linkis.governance.common.protocol.conf.{ RequestQueryEngineConfigWithGlobalConfig, ResponseQueryConfig } import org.apache.linkis.manager.label.entity.Label -import org.apache.linkis.manager.label.entity.engine.{ - ConcurrentEngineConnLabel, - EngineTypeLabel, - UserCreatorLabel -} +import org.apache.linkis.manager.label.entity.engine.{EngineTypeLabel, UserCreatorLabel} import org.apache.linkis.manager.label.utils.LabelUtil -import org.apache.linkis.protocol.constants.TaskConstant -import org.apache.linkis.protocol.utils.TaskUtils import org.apache.linkis.rpc.Sender import org.apache.linkis.scheduler.queue.{Group, GroupFactory, SchedulerEvent} import org.apache.linkis.scheduler.queue.parallelqueue.ParallelGroup @@ -46,8 +41,6 @@ import java.util import java.util.concurrent.TimeUnit import java.util.regex.Pattern -import scala.collection.JavaConverters._ - import com.google.common.cache.{Cache, CacheBuilder} class EntranceGroupFactory extends GroupFactory with Logging { @@ -58,7 +51,7 @@ class EntranceGroupFactory extends GroupFactory with Logging { .maximumSize(EntranceConfiguration.GROUP_CACHE_MAX.getValue) .build() - private val GROUP_MAX_CAPACITY = CommonVars("wds.linkis.entrance.max.capacity", 2000) + private val GROUP_MAX_CAPACITY = CommonVars("wds.linkis.entrance.max.capacity", 1000) private val SPECIFIED_USERNAME_REGEX = CommonVars("wds.linkis.entrance.specified.username.regex", "hduser.*") @@ -76,29 +69,16 @@ class EntranceGroupFactory extends GroupFactory with Logging { } override def getOrCreateGroup(event: SchedulerEvent): Group = { - val (labels, params) = event match { + val labels = event match { case job: EntranceJob => - (job.getJobRequest.getLabels, job.getJobRequest.getParams) + job.getJobRequest.getLabels + case _ => + throw new EntranceErrorException(LABEL_NOT_NULL.getErrorCode, LABEL_NOT_NULL.getErrorDesc) } - val groupName = EntranceGroupFactory.getGroupNameByLabels(labels, params) + val groupName = EntranceGroupFactory.getGroupNameByLabels(labels) val cacheGroup = groupNameToGroups.getIfPresent(groupName) if (null == cacheGroup) synchronized { val maxAskExecutorTimes = EntranceConfiguration.MAX_ASK_EXECUTOR_TIME.getValue.toLong - if (groupName.startsWith(EntranceGroupFactory.CONCURRENT)) { - if (null == groupNameToGroups.getIfPresent(groupName)) synchronized { - if (null == groupNameToGroups.getIfPresent(groupName)) { - val group = new ParallelGroup( - groupName, - 100, - EntranceConfiguration.CONCURRENT_FACTORY_MAX_CAPACITY.getValue - ) - group.setMaxRunningJobs(EntranceConfiguration.CONCURRENT_MAX_RUNNING_JOBS.getValue) - group.setMaxAskExecutorTimes(EntranceConfiguration.CONCURRENT_EXECUTOR_TIME.getValue) - groupNameToGroups.put(groupName, group) - return group - } - } - } val sender: Sender = Sender.getSender(Configuration.CLOUD_CONSOLE_CONFIGURATION_SPRING_APPLICATION_NAME.getValue) val userCreatorLabel: UserCreatorLabel = LabelUtil.getUserCreatorLabel(labels) @@ -136,8 +116,11 @@ class EntranceGroupFactory extends GroupFactory with Logging { group.setMaxRunningJobs(maxRunningJobs) group.setMaxAskExecutorTimes(maxAskExecutorTimes) groupNameToGroups.put(groupName, group) + group + } + else { + cacheGroup } - groupNameToGroups.getIfPresent(groupName) } override def getGroup(groupName: String): Group = { @@ -151,10 +134,18 @@ class EntranceGroupFactory extends GroupFactory with Logging { group } + /** + * User task concurrency control is controlled for multiple Entrances, which will be evenly + * distributed based on the number of existing Entrances + * @param keyAndValue + * @return + */ private def getUserMaxRunningJobs(keyAndValue: util.Map[String, String]): Int = { + val userDefinedRunningJobs = EntranceConfiguration.WDS_LINKIS_INSTANCE.getValue(keyAndValue) + val entranceNum = EntranceUtils.getRunningEntranceNumber() Math.max( EntranceConfiguration.ENTRANCE_INSTANCE_MIN.getValue, - EntranceConfiguration.WDS_LINKIS_INSTANCE.getValue(keyAndValue) + userDefinedRunningJobs / entranceNum ) } @@ -162,64 +153,21 @@ class EntranceGroupFactory extends GroupFactory with Logging { object EntranceGroupFactory { - val CACHE = "_Cache" - - val CONCURRENT = "Concurrent_" - - def getGroupName( - creator: String, - user: String, - params: util.Map[String, AnyRef] = new util.HashMap[String, AnyRef] - ): String = { - val runtime = TaskUtils.getRuntimeMap(params) - val cache = - if ( - runtime.get(TaskConstant.READ_FROM_CACHE) != null && runtime - .get(TaskConstant.READ_FROM_CACHE) - .asInstanceOf[Boolean] - ) { - CACHE - } else "" - if (StringUtils.isNotEmpty(creator)) creator + "_" + user + cache - else EntranceConfiguration.DEFAULT_REQUEST_APPLICATION_NAME.getValue + "_" + user + cache - } - - def getGroupNameByLabels( - labels: java.util.List[Label[_]], - params: util.Map[String, AnyRef] = new util.HashMap[String, AnyRef] - ): String = { - - val userCreator = labels.asScala.find(_.isInstanceOf[UserCreatorLabel]) - val engineType = labels.asScala.find(_.isInstanceOf[EngineTypeLabel]) - val concurrent = labels.asScala.find(_.isInstanceOf[ConcurrentEngineConnLabel]) - if (userCreator.isEmpty || engineType.isEmpty) { + /** + * Entrance group rule creator_username_engineType eg:IDE_PEACEWONG_SPARK + * @param labels + * @param params + * @return + */ + def getGroupNameByLabels(labels: java.util.List[Label[_]]): String = { + val userCreatorLabel = LabelUtil.getUserCreatorLabel(labels) + val engineTypeLabel = LabelUtil.getEngineTypeLabel(labels) + if (null == userCreatorLabel || null == engineTypeLabel) { throw new EntranceErrorException(LABEL_NOT_NULL.getErrorCode, LABEL_NOT_NULL.getErrorDesc) } - - if (concurrent.isDefined) { - - val engineTypeLabel = engineType.get.asInstanceOf[EngineTypeLabel] - val groupName = CONCURRENT + engineTypeLabel.getEngineType - groupName - - } else { - val userCreatorLabel = userCreator.get.asInstanceOf[UserCreatorLabel] - - val engineTypeLabel = engineType.get.asInstanceOf[EngineTypeLabel] - - val runtime = TaskUtils.getRuntimeMap(params) - val cache = - if ( - runtime.get(TaskConstant.READ_FROM_CACHE) != null && runtime - .get(TaskConstant.READ_FROM_CACHE) - .asInstanceOf[Boolean] - ) { - CACHE - } else "" - val groupName = - userCreatorLabel.getCreator + "_" + userCreatorLabel.getUser + "_" + engineTypeLabel.getEngineType + cache - groupName - } + val groupName = + userCreatorLabel.getCreator + "_" + userCreatorLabel.getUser + "_" + engineTypeLabel.getEngineType + groupName } } diff --git a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/scheduler/EntranceParallelConsumerManager.scala b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/scheduler/EntranceParallelConsumerManager.scala index 2cdee97cc7..789e2ca2b1 100644 --- a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/scheduler/EntranceParallelConsumerManager.scala +++ b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/scheduler/EntranceParallelConsumerManager.scala @@ -20,6 +20,7 @@ package org.apache.linkis.entrance.scheduler import org.apache.linkis.common.ServiceInstance import org.apache.linkis.common.utils.Utils import org.apache.linkis.entrance.conf.EntranceConfiguration +import org.apache.linkis.entrance.utils.EntranceUtils import org.apache.linkis.instance.label.client.InstanceLabelClient import org.apache.linkis.manager.label.builder.factory.LabelBuilderFactoryContext import org.apache.linkis.manager.label.constant.{LabelKeyConstant, LabelValueConstant} @@ -45,29 +46,10 @@ class EntranceParallelConsumerManager(maxParallelismUsers: Int, schedulerName: S if (EntranceConfiguration.ENTRANCE_GROUP_SCAN_ENABLED.getValue) { Utils.defaultScheduler.scheduleAtFixedRate( new Runnable { - override def run(): Unit = { - Utils.tryAndError { - logger.info("start refresh consumer group maxAllowRunningJobs") - // get all entrance server from eureka - val serviceInstances = - Sender.getInstances(Sender.getThisServiceInstance.getApplicationName) - if (null == serviceInstances || serviceInstances.isEmpty) return - - // get all offline label server - val routeLabel = LabelBuilderFactoryContext.getLabelBuilderFactory - .createLabel[RouteLabel](LabelKeyConstant.ROUTE_KEY, LabelValueConstant.OFFLINE_VALUE) - val labels = new util.ArrayList[Label[_]] - labels.add(routeLabel) - val labelInstances = InstanceLabelClient.getInstance.getInstanceFromLabel(labels) - - // get active entrance server - val allInstances = new util.ArrayList[ServiceInstance]() - allInstances.addAll(serviceInstances.toList.asJava) - allInstances.removeAll(labelInstances) - // refresh all group maxAllowRunningJobs - refreshAllGroupMaxAllowRunningJobs(allInstances.size()) - logger.info("Finished to refresh consumer group maxAllowRunningJobs") - } + override def run(): Unit = Utils.tryAndWarn { + // refresh all group maxAllowRunningJobs + refreshAllGroupMaxAllowRunningJobs(EntranceUtils.getRunningEntranceNumber()) + logger.info("Finished to refresh consumer group maxAllowRunningJobs") } }, EntranceConfiguration.ENTRANCE_GROUP_SCAN_INIT_TIME.getValue, diff --git a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/scheduler/cache/ReadCacheConsumer.scala b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/scheduler/cache/ReadCacheConsumer.scala deleted file mode 100644 index 65bbbd39b4..0000000000 --- a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/scheduler/cache/ReadCacheConsumer.scala +++ /dev/null @@ -1,158 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.entrance.scheduler.cache - -import org.apache.linkis.common.io.FsPath -import org.apache.linkis.common.utils.Utils -import org.apache.linkis.entrance.errorcode.EntranceErrorCodeSummary._ -import org.apache.linkis.entrance.exception.CacheNotReadyException -import org.apache.linkis.entrance.execute.EntranceJob -import org.apache.linkis.entrance.persistence.PersistenceManager -import org.apache.linkis.entrance.utils.JobHistoryHelper -import org.apache.linkis.governance.common.entity.job.JobRequest -import org.apache.linkis.manager.label.constant.LabelKeyConstant -import org.apache.linkis.protocol.constants.TaskConstant -import org.apache.linkis.protocol.utils.TaskUtils -import org.apache.linkis.scheduler.SchedulerContext -import org.apache.linkis.scheduler.errorcode.LinkisSchedulerErrorCodeSummary._ -import org.apache.linkis.scheduler.exception.SchedulerErrorException -import org.apache.linkis.scheduler.executer.SuccessExecuteResponse -import org.apache.linkis.scheduler.queue.Group -import org.apache.linkis.scheduler.queue.fifoqueue.FIFOUserConsumer -import org.apache.linkis.server.BDPJettyServerHelper -import org.apache.linkis.storage.FSFactory -import org.apache.linkis.storage.fs.FileSystem - -import org.apache.commons.io.FilenameUtils -import org.apache.commons.lang3.StringUtils - -import java.util.concurrent.ExecutorService - -import scala.collection.JavaConverters._ - -import com.google.common.collect.Lists - -class ReadCacheConsumer( - schedulerContext: SchedulerContext, - executeService: ExecutorService, - private var group: Group, - persistenceManager: PersistenceManager -) extends FIFOUserConsumer(schedulerContext, executeService, group) { - - override protected def loop(): Unit = { - val event = Option(getConsumeQueue.take()) - event.foreach { - case job: EntranceJob => - job.getJobRequest match { - case jobRequest: JobRequest => - Utils.tryCatch { - val engineTpyeLabel = jobRequest.getLabels.asScala - .filter(l => l.getLabelKey.equalsIgnoreCase(LabelKeyConstant.ENGINE_TYPE_KEY)) - .headOption - .getOrElse(null) - val labelStrList = jobRequest.getLabels.asScala.map { case l => - l.getStringValue - }.toList - if (null == engineTpyeLabel) { - logger.error( - "Invalid engineType null, cannot process. jobReq : " + BDPJettyServerHelper.gson - .toJson(jobRequest) - ) - throw CacheNotReadyException( - INVALID_ENGINETYPE_NULL.getErrorCode, - INVALID_ENGINETYPE_NULL.getErrorDesc - ) - } - val readCacheBefore: Long = TaskUtils - .getRuntimeMap(job.getParams) - .getOrDefault(TaskConstant.READ_CACHE_BEFORE, 300L: java.lang.Long) - .asInstanceOf[Long] - val cacheResult = JobHistoryHelper.getCache( - jobRequest.getExecutionCode, - jobRequest.getExecuteUser, - labelStrList.asJava, - readCacheBefore - ) - if (cacheResult != null && StringUtils.isNotBlank(cacheResult.getResultLocation)) { - val resultSets = listResults(cacheResult.getResultLocation, job.getUser) - if (resultSets.size() > 0) { - for (resultSet: FsPath <- resultSets.asScala) { - val alias = FilenameUtils.getBaseName(resultSet.getPath) - val output = FsPath - .getFsPath( - cacheResult.getResultLocation, - FilenameUtils.getName(resultSet.getPath) - ) - .getSchemaPath -// persistenceManager.onResultSetCreated(job, new CacheOutputExecuteResponse(alias, output)) - throw CacheNotReadyException( - INVALID_RESULTSETS.getErrorCode, - INVALID_RESULTSETS.getErrorDesc - ) - // todo check - } -// persistenceManager.onResultSizeCreated(job, resultSets.size()) - } - val runtime = TaskUtils.getRuntimeMap(job.getParams) - runtime.put(TaskConstant.CACHE, java.lang.Boolean.FALSE) - TaskUtils.addRuntimeMap(job.getParams, runtime) - job.transitionCompleted(SuccessExecuteResponse(), "Result found in cache") - } else { - logger.info("Cache not found, submit to normal consumer.") - submitToExecute(job) - } - } { t => - logger.warn("Read cache failed, submit to normal consumer: ", t) - submitToExecute(job) - } - case _ => - } - case _ => - } - } - - private def listResults(resultLocation: String, user: String) = { - val dirPath = FsPath.getFsPath(resultLocation) - val fileSystem = FSFactory.getFsByProxyUser(dirPath, user).asInstanceOf[FileSystem] - Utils.tryFinally { - fileSystem.init(null) - if (fileSystem.exists(dirPath)) { - fileSystem.listPathWithError(dirPath).getFsPaths - } else { - Lists.newArrayList[FsPath]() - } - }(Utils.tryQuietly(fileSystem.close())) - } - - private def submitToExecute(job: EntranceJob): Unit = { - val runtime = TaskUtils.getRuntimeMap(job.getParams) - runtime.put(TaskConstant.READ_FROM_CACHE, java.lang.Boolean.FALSE) - TaskUtils.addRuntimeMap(job.getParams, runtime) - val groupName = schedulerContext.getOrCreateGroupFactory.getOrCreateGroup(job).getGroupName - val consumer = schedulerContext.getOrCreateConsumerManager.getOrCreateConsumer(groupName) - val index = consumer.getConsumeQueue.offer(job) - // index.map(getEventId(_, groupName)).foreach(job.setId) - if (index.isEmpty) { - throw new SchedulerErrorException( - JOB_QUEUE_IS_FULL.getErrorCode, - JOB_QUEUE_IS_FULL.getErrorDesc - ) - } - } - -} diff --git a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/scheduler/cache/ReadCacheConsumerManager.scala b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/scheduler/cache/ReadCacheConsumerManager.scala deleted file mode 100644 index a4cba19f34..0000000000 --- a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/scheduler/cache/ReadCacheConsumerManager.scala +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.entrance.scheduler.cache - -import org.apache.linkis.entrance.persistence.PersistenceManager -import org.apache.linkis.entrance.scheduler.EntranceGroupFactory -import org.apache.linkis.scheduler.queue.fifoqueue.FIFOUserConsumer -import org.apache.linkis.scheduler.queue.parallelqueue.ParallelConsumerManager - -class ReadCacheConsumerManager(maxParallelismUsers: Int, persistenceManager: PersistenceManager) - extends ParallelConsumerManager(maxParallelismUsers) { - - override protected def createConsumer(groupName: String): FIFOUserConsumer = { - val group = getSchedulerContext.getOrCreateGroupFactory.getGroup(groupName) - if (groupName.endsWith(EntranceGroupFactory.CACHE)) { - logger.info("Create cache consumer with group: " + groupName) - new ReadCacheConsumer( - getSchedulerContext, - getOrCreateExecutorService, - group, - persistenceManager - ) - } else { - logger.info("Create normal consumer with group: " + groupName) - new FIFOUserConsumer(getSchedulerContext, getOrCreateExecutorService, group) - } - } - -} diff --git a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/timeout/JobTimeoutManager.scala b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/timeout/JobTimeoutManager.scala index aaaf131bd8..4e62430316 100644 --- a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/timeout/JobTimeoutManager.scala +++ b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/timeout/JobTimeoutManager.scala @@ -38,8 +38,8 @@ class JobTimeoutManager extends Logging { private[this] final val timeoutJobByName: ConcurrentMap[String, EntranceJob] = new ConcurrentHashMap[String, EntranceJob] - val timeoutCheck: Boolean = EntranceConfiguration.ENABLE_JOB_TIMEOUT_CHECK.getValue - val timeoutScanInterval: Int = EntranceConfiguration.TIMEOUT_SCAN_INTERVAL.getValue + private val timeoutCheck: Boolean = EntranceConfiguration.ENABLE_JOB_TIMEOUT_CHECK.getValue + private val timeoutScanInterval: Int = EntranceConfiguration.TIMEOUT_SCAN_INTERVAL.getValue def add(jobKey: String, job: EntranceJob): Unit = { logger.info(s"Adding timeout job: ${job.getId()}") @@ -77,75 +77,75 @@ class JobTimeoutManager extends Logging { } private def timeoutDetective(): Unit = { - if (timeoutCheck) { - def checkAndSwitch(job: EntranceJob): Unit = { - logger.info(s"Checking whether the job id ${job.getJobRequest.getId()} timed out. ") - val currentTimeSeconds = System.currentTimeMillis() / 1000 - // job.isWaiting == job in queue - val jobScheduleStartTimeSeconds = - if (job.isWaiting) job.createTime / 1000 else currentTimeSeconds - val queuingTimeSeconds = currentTimeSeconds - jobScheduleStartTimeSeconds - val jobRunningStartTimeSeconds = - if (job.getStartTime > 0) job.getStartTime / 1000 else currentTimeSeconds - val runningTimeSeconds = currentTimeSeconds - jobRunningStartTimeSeconds - if (!job.isCompleted) { - job.jobRequest.getLabels.asScala foreach { - case queueTimeOutLabel: JobQueuingTimeoutLabel => - if ( - job.isWaiting && queueTimeOutLabel.getQueuingTimeout > 0 && queuingTimeSeconds >= queueTimeOutLabel.getQueuingTimeout - ) { - logger.warn( - s"Job ${job.getJobRequest.getId()} queued time : ${queuingTimeSeconds} seconds, which was over queueTimeOut : ${queueTimeOutLabel.getQueuingTimeout} seconds, cancel it now! " - ) - job.onFailure( - s"Job queued ${queuingTimeSeconds} seconds over max queue time : ${queueTimeOutLabel.getQueuingTimeout} seconds.", - null - ) - } - case jobRunningTimeoutLabel: JobRunningTimeoutLabel => - if ( - job.isRunning && jobRunningTimeoutLabel.getRunningTimeout > 0 && runningTimeSeconds >= jobRunningTimeoutLabel.getRunningTimeout - ) { - logger.warn( - s"Job ${job.getJobRequest.getId()} run timeout ${runningTimeSeconds} seconds, which was over runTimeOut : ${jobRunningTimeoutLabel.getRunningTimeout} seconds, cancel it now! " - ) - job.onFailure( - s"Job run ${runningTimeSeconds} seconds over max run time : ${jobRunningTimeoutLabel.getRunningTimeout} seconds.", - null - ) - } - case _ => - } + def checkAndSwitch(job: EntranceJob): Unit = { + logger.info(s"Checking whether the job id ${job.getJobRequest.getId()} timed out. ") + val currentTimeSeconds = System.currentTimeMillis() / 1000 + // job.isWaiting == job in queue + val jobScheduleStartTimeSeconds = + if (job.isWaiting) job.createTime / 1000 else currentTimeSeconds + val queuingTimeSeconds = currentTimeSeconds - jobScheduleStartTimeSeconds + val jobRunningStartTimeSeconds = + if (job.getStartTime > 0) job.getStartTime / 1000 else currentTimeSeconds + val runningTimeSeconds = currentTimeSeconds - jobRunningStartTimeSeconds + if (!job.isCompleted) { + job.jobRequest.getLabels.asScala foreach { + case queueTimeOutLabel: JobQueuingTimeoutLabel => + if ( + job.isWaiting && queueTimeOutLabel.getQueuingTimeout > 0 && queuingTimeSeconds >= queueTimeOutLabel.getQueuingTimeout + ) { + logger.warn( + s"Job ${job.getJobRequest.getId()} queued time : ${queuingTimeSeconds} seconds, which was over queueTimeOut : ${queueTimeOutLabel.getQueuingTimeout} seconds, cancel it now! " + ) + job.onFailure( + s"Job queued ${queuingTimeSeconds} seconds over max queue time : ${queueTimeOutLabel.getQueuingTimeout} seconds.", + null + ) + } + case jobRunningTimeoutLabel: JobRunningTimeoutLabel => + if ( + job.isRunning && jobRunningTimeoutLabel.getRunningTimeout > 0 && runningTimeSeconds >= jobRunningTimeoutLabel.getRunningTimeout + ) { + logger.warn( + s"Job ${job.getJobRequest.getId()} run timeout ${runningTimeSeconds} seconds, which was over runTimeOut : ${jobRunningTimeoutLabel.getRunningTimeout} seconds, cancel it now! " + ) + job.onFailure( + s"Job run ${runningTimeSeconds} seconds over max run time : ${jobRunningTimeoutLabel.getRunningTimeout} seconds.", + null + ) + } + case _ => } } - - timeoutJobByName.asScala.foreach(item => { - logger.info(s"Running timeout detection!") - synchronized { - jobCompleteDelete(item._1) - if (jobExist(item._1)) checkAndSwitch(item._2) - } - }) } + + timeoutJobByName.asScala.foreach(item => { + logger.info(s"Running timeout detection!") + synchronized { + jobCompleteDelete(item._1) + if (jobExist(item._1)) checkAndSwitch(item._2) + } + }) } // Thread periodic scan timeout task - val woker = Utils.defaultScheduler.scheduleAtFixedRate( - new Runnable() { - - override def run(): Unit = { - Utils.tryCatch { - timeoutDetective() - } { case t: Throwable => - logger.error(s"TimeoutDetective task failed. ${t.getMessage}", t) + if (timeoutCheck) { + val woker = Utils.defaultScheduler.scheduleAtFixedRate( + new Runnable() { + + override def run(): Unit = { + Utils.tryCatch { + timeoutDetective() + } { case t: Throwable => + logger.warn(s"TimeoutDetective task failed. ${t.getMessage}", t) + } } - } - }, - 0, - timeoutScanInterval, - TimeUnit.SECONDS - ) + }, + 0, + timeoutScanInterval, + TimeUnit.SECONDS + ) + } } diff --git a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/utils/EntranceUtils.scala b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/utils/EntranceUtils.scala new file mode 100644 index 0000000000..13dcefa9f9 --- /dev/null +++ b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/utils/EntranceUtils.scala @@ -0,0 +1,108 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.entrance.utils + +import org.apache.linkis.common.ServiceInstance +import org.apache.linkis.common.utils.{Logging, Utils} +import org.apache.linkis.instance.label.client.InstanceLabelClient +import org.apache.linkis.manager.label.builder.factory.LabelBuilderFactoryContext +import org.apache.linkis.manager.label.constant.{LabelKeyConstant, LabelValueConstant} +import org.apache.linkis.manager.label.entity.Label +import org.apache.linkis.manager.label.entity.engine.{EngineTypeLabel, UserCreatorLabel} +import org.apache.linkis.manager.label.entity.route.RouteLabel +import org.apache.linkis.manager.label.utils.EngineTypeLabelCreator +import org.apache.linkis.rpc.Sender + +import org.apache.commons.lang3.StringUtils + +import java.util + +import scala.collection.JavaConverters.asScalaBufferConverter + +object EntranceUtils extends Logging { + + private val SPLIT = "," + + private val labelFactory = LabelBuilderFactoryContext.getLabelBuilderFactory + + def getUserCreatorEcTypeKey( + userCreatorLabel: UserCreatorLabel, + engineTypeLabel: EngineTypeLabel + ): String = { + userCreatorLabel.getStringValue + SPLIT + engineTypeLabel.getStringValue + } + + def fromKeyGetLabels(key: String): (UserCreatorLabel, EngineTypeLabel) = { + if (StringUtils.isBlank(key)) (null, null) + else { + val labelStringValues = key.split(SPLIT) + if (labelStringValues.length < 2) return (null, null) + val userCreatorLabel = labelFactory + .createLabel[UserCreatorLabel](LabelKeyConstant.USER_CREATOR_TYPE_KEY, labelStringValues(0)) + val engineTypeLabel = labelFactory + .createLabel[EngineTypeLabel](LabelKeyConstant.ENGINE_TYPE_KEY, labelStringValues(1)) + (userCreatorLabel, engineTypeLabel) + } + } + + def getDefaultCreatorECTypeKey(creator: String, ecType: String): String = { + val userCreatorLabel = + labelFactory.createLabel[UserCreatorLabel](LabelKeyConstant.USER_CREATOR_TYPE_KEY) + val ecTypeLabel = EngineTypeLabelCreator.createEngineTypeLabel(ecType) + userCreatorLabel.setUser("*") + userCreatorLabel.setCreator(creator) + getUserCreatorEcTypeKey(userCreatorLabel, ecTypeLabel) + } + + def getRunningEntranceNumber(): Int = { + val entranceNum = Sender.getInstances(Sender.getThisServiceInstance.getApplicationName).length + val labelList = new util.ArrayList[Label[_]]() + val offlineRouteLabel = LabelBuilderFactoryContext.getLabelBuilderFactory + .createLabel[RouteLabel](LabelKeyConstant.ROUTE_KEY, LabelValueConstant.OFFLINE_VALUE) + labelList.add(offlineRouteLabel) + var offlineIns: Array[ServiceInstance] = null + Utils.tryAndWarn { + offlineIns = InstanceLabelClient.getInstance + .getInstanceFromLabel(labelList) + .asScala + .filter(l => + null != l && l.getApplicationName + .equalsIgnoreCase(Sender.getThisServiceInstance.getApplicationName) + ) + .toArray + } + val entranceRealNumber = if (null != offlineIns) { + logger.info(s"There are ${offlineIns.length} offlining instance.") + entranceNum - offlineIns.length + } else { + entranceNum + } + /* + Sender.getInstances may get 0 instances due to cache in Sender. So this instance is the one instance. + */ + if (entranceRealNumber <= 0) { + logger.error( + s"Got ${entranceRealNumber} ${Sender.getThisServiceInstance.getApplicationName} instances." + ) + 1 + } else { + entranceRealNumber + } + } + +} diff --git a/linkis-computation-governance/linkis-entrance/src/test/java/org/apache/linkis/entrance/interceptor/impl/SQLExplainTest.java b/linkis-computation-governance/linkis-entrance/src/test/java/org/apache/linkis/entrance/interceptor/impl/SQLExplainTest.java new file mode 100644 index 0000000000..c5efb5633e --- /dev/null +++ b/linkis-computation-governance/linkis-entrance/src/test/java/org/apache/linkis/entrance/interceptor/impl/SQLExplainTest.java @@ -0,0 +1,55 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.entrance.interceptor.impl; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +class SQLExplainTest { + + @Test + void isSelectCmdNoLimit() { + + String code = "SELECT * from dual WHERE (1=1)LIMIT 1;"; + boolean res = SQLExplain.isSelectCmdNoLimit(code); + Assertions.assertEquals(false, res); + + code = "SELECT * from dual"; + res = SQLExplain.isSelectCmdNoLimit(code); + Assertions.assertEquals(true, res); + + code = "SELECT * from dual LIMIT 1;"; + res = SQLExplain.isSelectCmdNoLimit(code); + Assertions.assertEquals(false, res); + } + + @Test + void isSelectOverLimit() { + String code = "SELECT * from dual WHERE (1=1)LIMIT 5001;"; + boolean res = SQLExplain.isSelectOverLimit(code); + Assertions.assertEquals(true, res); + + code = "SELECT * from dual"; + res = SQLExplain.isSelectOverLimit(code); + Assertions.assertEquals(false, res); + + code = "SELECT * from dual LIMIT 4000;"; + res = SQLExplain.isSelectOverLimit(code); + Assertions.assertEquals(false, res); + } +} diff --git a/linkis-computation-governance/linkis-entrance/src/test/java/org/apache/linkis/entrance/interceptor/impl/TemplateConfUtilsTest.java b/linkis-computation-governance/linkis-entrance/src/test/java/org/apache/linkis/entrance/interceptor/impl/TemplateConfUtilsTest.java new file mode 100644 index 0000000000..c965529b57 --- /dev/null +++ b/linkis-computation-governance/linkis-entrance/src/test/java/org/apache/linkis/entrance/interceptor/impl/TemplateConfUtilsTest.java @@ -0,0 +1,128 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.entrance.interceptor.impl; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +class TemplateConfUtilsTest { + + @Test + void getCustomTemplateConfName() { + String sqlCode = + "" + + "--注解\n" + + "select * from table;\n" + + " --注解 \n" + + "--注解\n" + + " select \"--注解\" as test\n" + + " --@set yy=123\n" + + " --注解"; + + String res = TemplateConfUtils.getCustomTemplateConfName(sqlCode, "sql"); + Assertions.assertEquals(res, ""); + + String sqlCode2 = + "" + + "---@set 123=注解\n" + + "select * from table;\n" + + " --注解 \n" + + "--注解\n" + + " select \"--注解\" as test\n" + + " --@set yy=123\n" + + " --注解"; + + res = TemplateConfUtils.getCustomTemplateConfName(sqlCode2, "sql"); + Assertions.assertEquals(res, ""); + + String sqlCode3 = + "" + + "---@set ec.resource.name=345\n" + + "select * from table;\n" + + " --注解 \n" + + "--注解\n" + + "---@set ec.resource.name=456\n" + + " select \"--注解\" as test\n" + + " --@set yy=123\n" + + " --注解"; + + res = TemplateConfUtils.getCustomTemplateConfName(sqlCode3, "sql"); + Assertions.assertEquals(res, "345"); + + String sqlCode4 = + "" + + "---@set ec.resource.name= name1 \n" + + " select \"--注解\" as test\n" + + " --@set yy=123\n" + + " --注解"; + + res = TemplateConfUtils.getCustomTemplateConfName(sqlCode4, "sql"); + Assertions.assertEquals(res, "name1"); + + String sqlCode5 = + "" + + "##@set ec.resource.name=pyname1\n" + + "select * from table;\n" + + " --注解 \n" + + "#注解\n" + + "##@set ec.resource.name= 123 \n" + + " select \"--注解\" as test\n" + + "#@set yy=123\n" + + " #注解"; + + res = TemplateConfUtils.getCustomTemplateConfName(sqlCode5, "python"); + Assertions.assertEquals(res, "pyname1"); + + String sqlCode6 = + "" + + "///@set ec.resource.name= scalaname1 \n" + + " select \"//注解\" as test\n" + + "//@set yy=123\n" + + " #注解"; + + res = TemplateConfUtils.getCustomTemplateConfName(sqlCode6, "scala"); + Assertions.assertEquals(res, "scalaname1"); + + String sqlCode7 = + "" + + "---@set ec.resource.name= hqlname1 \n" + + " select \"--注解\" as test\n" + + " --@set yy=123\n" + + " --注解"; + + res = TemplateConfUtils.getCustomTemplateConfName(sqlCode7, "hql"); + Assertions.assertEquals(res, "hqlname1"); + + String sqlCode8 = + "---@set ec.resource.name=linkis_test2;\n" + + " ---@set ec.resource.name=scriptis_test hive;\n" + + " select * from dss autotest.demo data limit 100;"; + res = TemplateConfUtils.getCustomTemplateConfName(sqlCode8, "hql"); + Assertions.assertEquals(res, "linkis_test2"); + } + + @Test + void getCustomTemplateConfName2() { + + String sqlCode9 = + "---@set ec.resource.name=linkis_test2;\r\n---@set ec.resource.name=scriptis_test_hive;\r\n--@set limitn=100\r\nselect * from dss_autotest.demo_data limit ${limitn};\r\n"; + + String res = TemplateConfUtils.getCustomTemplateConfName(sqlCode9, "hql"); + Assertions.assertEquals(res, "linkis_test2"); + } +} diff --git a/linkis-computation-governance/linkis-entrance/src/test/java/org/apache/linkis/entrance/interceptor/impl/TestCommentHelper.java b/linkis-computation-governance/linkis-entrance/src/test/java/org/apache/linkis/entrance/interceptor/impl/TestCommentHelper.java new file mode 100644 index 0000000000..622d06c4e8 --- /dev/null +++ b/linkis-computation-governance/linkis-entrance/src/test/java/org/apache/linkis/entrance/interceptor/impl/TestCommentHelper.java @@ -0,0 +1,65 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.entrance.interceptor.impl; + +import java.util.Arrays; + +import org.junit.jupiter.api.Test; +import org.junit.platform.commons.util.StringUtils; + +public class TestCommentHelper { + String sqlCode = + "" + + "--注解\n" + + "select * from table;\n" + + " --注解 \n" + + "--注解\n" + + " select \"--注解\" as test\n" + + " --@set yy=123\n" + + " --注解"; + + String scalaCode = + "" + + "// 注解\n" + + "print(1+1)\n" + + "//@set yy=123\n" + + " print(2)\n" + + " // 注解 \n" + + "// test\n" + + "print(\"//注解测试\")"; + + String scalaCodeRes = "print(1+1)\n" + "print(2)\n" + "print(\"//注解测试\")"; + + @Test + void sqlDealCommentTest() { + String code = SQLCommentHelper.dealComment(sqlCode); + // System.out.println(code); + } + + @Test + void scalaDealCommentTest() { + String code = ScalaCommentHelper.dealComment(scalaCode); + String[] lines = + Arrays.stream(code.split("\n")) + .map(String::trim) + .filter(x -> StringUtils.isNotBlank(x)) + .toArray(String[]::new); + String result = String.join("\n", lines); + // assertEquals(result,scalaCodeRes); + } +} diff --git a/linkis-computation-governance/linkis-entrance/src/test/java/org/apache/linkis/entrance/interceptor/impl/TestHDFSCacheLogWriter.java b/linkis-computation-governance/linkis-entrance/src/test/java/org/apache/linkis/entrance/interceptor/impl/TestHDFSCacheLogWriter.java new file mode 100644 index 0000000000..fabff88473 --- /dev/null +++ b/linkis-computation-governance/linkis-entrance/src/test/java/org/apache/linkis/entrance/interceptor/impl/TestHDFSCacheLogWriter.java @@ -0,0 +1,75 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.entrance.interceptor.impl; + +import org.apache.linkis.entrance.log.Cache; +import org.apache.linkis.entrance.log.HDFSCacheLogWriter; + +import org.apache.commons.lang3.StringUtils; + +import java.io.File; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.UUID; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import oshi.util.FileUtil; + +class TestHDFSCacheLogWriter { + + @Test + void write() throws IOException { + + Cache cache = new Cache(5); + String fileName = UUID.randomUUID().toString().replace("-", "") + "-test.log"; + String logPath = System.getProperty("java.io.tmpdir") + File.separator + fileName; + System.out.println(logPath); + String chartSet = "utf-8"; + String username = System.getProperty("user.name"); + + File file = new File(logPath); + file.createNewFile(); + + HDFSCacheLogWriter logWriter = + new HDFSCacheLogWriter( + // "D:\\DataSphere\\linkis\\docs\\test.log", + logPath, chartSet, cache, username); + + String[] msgArr = + new String[] { + "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15", "16", + "17", "18" + }; + + List msgList = new ArrayList(Arrays.asList(msgArr)); + String msg = String.join("\n", msgList); + + logWriter.write(msg); + logWriter.flush(); + + List list = FileUtil.readFile(logPath); + String res = String.join("\n", list); + + res = res.replace("\n\n", "\n"); + res = StringUtils.strip(res, " \n"); + Assertions.assertEquals(res, msg); + } +} diff --git a/linkis-computation-governance/linkis-entrance/src/test/scala/org/apache/linkis/entrance/interceptor/impl/TestReplaceComment.scala b/linkis-computation-governance/linkis-entrance/src/test/scala/org/apache/linkis/entrance/interceptor/impl/TestReplaceComment.scala new file mode 100644 index 0000000000..3310f09581 --- /dev/null +++ b/linkis-computation-governance/linkis-entrance/src/test/scala/org/apache/linkis/entrance/interceptor/impl/TestReplaceComment.scala @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.entrance.interceptor.impl + +import org.junit.jupiter.api.{Assertions, Test} + +class TestReplaceComment { + + @Test + def TestRepComm: Unit = { + val realCode = "drop table if exists default.test;" + + "create table default.test(" + + "id varchar(11) comment '这是注释测试分号;这是注释测试分号;'," + + "id1 string comment '测试'," + + "id2 string COMMENT '码值说明:2-高;3-中;4-低;'" + + ");" + val expectCode = "drop table if exists default.test;" + + "create table default.test(" + + "id varchar(11) comment '这是注释测试分号\\;这是注释测试分号\\;'," + + "id1 string comment '测试'," + + "id2 string COMMENT '码值说明:2-高\\;3-中\\;4-低\\;'" + + ");" + + Assertions.assertEquals(SQLCommentHelper.replaceComment(realCode), expectCode) + } + +} diff --git a/linkis-computation-governance/linkis-jdbc-driver/src/main/scala/org/apache/linkis/ujes/jdbc/LinkisSQLConnection.scala b/linkis-computation-governance/linkis-jdbc-driver/src/main/scala/org/apache/linkis/ujes/jdbc/LinkisSQLConnection.scala index 7b3e2ada8c..80298bcb1b 100644 --- a/linkis-computation-governance/linkis-jdbc-driver/src/main/scala/org/apache/linkis/ujes/jdbc/LinkisSQLConnection.scala +++ b/linkis-computation-governance/linkis-jdbc-driver/src/main/scala/org/apache/linkis/ujes/jdbc/LinkisSQLConnection.scala @@ -451,6 +451,7 @@ class LinkisSQLConnection(private[jdbc] val ujesClient: UJESClient, props: Prope case EngineType.SPARK => RunType.SQL case EngineType.HIVE => RunType.HIVE case EngineType.REPL => RunType.REPL + case EngineType.DORIS => RunType.DORIS case EngineType.TRINO => RunType.TRINO_SQL case EngineType.PRESTO => RunType.PRESTO_SQL case EngineType.NEBULA => RunType.NEBULA_SQL diff --git a/linkis-computation-governance/linkis-jdbc-driver/src/main/scala/org/apache/linkis/ujes/jdbc/UJESSQLResultSet.scala b/linkis-computation-governance/linkis-jdbc-driver/src/main/scala/org/apache/linkis/ujes/jdbc/UJESSQLResultSet.scala index 0ed47925c6..dd5ace50e7 100644 --- a/linkis-computation-governance/linkis-jdbc-driver/src/main/scala/org/apache/linkis/ujes/jdbc/UJESSQLResultSet.scala +++ b/linkis-computation-governance/linkis-jdbc-driver/src/main/scala/org/apache/linkis/ujes/jdbc/UJESSQLResultSet.scala @@ -244,6 +244,8 @@ class UJESSQLResultSet( case null => throw new LinkisSQLException(LinkisSQLErrorCode.METADATA_EMPTY) case "char" | "varchar" | "nvarchar" | "string" => value case "short" => value.toShort + case "smallint" => value.toShort + case "tinyint" => value.toShort case "int" => value.toInt case "long" => value.toLong case "float" => value.toFloat diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/conf/AMConfiguration.java b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/conf/AMConfiguration.java index 0f018ca9de..80b433b3f2 100644 --- a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/conf/AMConfiguration.java +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/conf/AMConfiguration.java @@ -69,7 +69,7 @@ public class AMConfiguration { public static final CommonVars MULTI_USER_ENGINE_TYPES = CommonVars.apply( "wds.linkis.multi.user.engine.types", - "jdbc,es,presto,io_file,appconn,openlookeng,trino,nebula,hbase"); + "jdbc,es,presto,io_file,appconn,openlookeng,trino,nebula,hbase,doris"); public static final CommonVars ALLOW_BATCH_KILL_ENGINE_TYPES = CommonVars.apply("wds.linkis.allow.batch.kill.engine.types", "spark,hive,python"); @@ -105,8 +105,8 @@ public class AMConfiguration { public static String getDefaultMultiEngineUser() { String jvmUser = Utils.getJvmUser(); return String.format( - "{jdbc:\"%s\", es: \"%s\", presto:\"%s\", appconn:\"%s\", openlookeng:\"%s\", trino:\"%s\", nebula:\"%s\", hbase:\"%s\",io_file:\"root\"}", - jvmUser, jvmUser, jvmUser, jvmUser, jvmUser, jvmUser, jvmUser, jvmUser); + "{jdbc:\"%s\", es: \"%s\", presto:\"%s\", appconn:\"%s\", openlookeng:\"%s\", trino:\"%s\", nebula:\"%s\",doris:\"%s\", hbase:\"%s\",io_file:\"root\"}", + jvmUser, jvmUser, jvmUser, jvmUser, jvmUser, jvmUser, jvmUser, jvmUser, jvmUser); } public static boolean isMultiUserEngine(String engineType) { diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/restful/ECResourceInfoRestfulApi.java b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/restful/ECResourceInfoRestfulApi.java index 0ff9ef93eb..c3c14a8094 100644 --- a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/restful/ECResourceInfoRestfulApi.java +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/restful/ECResourceInfoRestfulApi.java @@ -237,8 +237,7 @@ public Message queryEcList(HttpServletRequest req, @RequestBody JsonNode jsonNod // check special admin token if (StringUtils.isNotBlank(token)) { if (!Configuration.isAdminToken(token)) { - logger.warn("Token:{} has no permission to query ecList.", token); - return Message.error("Token:" + token + " has no permission to query ecList."); + return Message.error("Token has no permission to query ecList."); } } else if (!Configuration.isAdmin(username)) { logger.warn("User:{} has no permission to query ecList.", username); diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/restful/EngineRestfulApi.java b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/restful/EngineRestfulApi.java index 14cad1380e..355ce2e9c6 100644 --- a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/restful/EngineRestfulApi.java +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/restful/EngineRestfulApi.java @@ -479,8 +479,7 @@ public Message killEngineAsyn(HttpServletRequest req, @RequestBody JsonNode json // check special token if (StringUtils.isNotBlank(token)) { if (!Configuration.isAdminToken(token)) { - logger.warn("Token {} has no permission to asyn kill engines.", token); - return Message.error("Token:" + token + " has no permission to asyn kill engines."); + return Message.error("Token has no permission to asyn kill engines."); } } else if (!Configuration.isAdmin(username)) { logger.warn("User {} has no permission to asyn kill engines.", username); diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/rm/utils/UserConfiguration.java b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/rm/utils/UserConfiguration.java index b7629bfa4e..123aa5995b 100644 --- a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/rm/utils/UserConfiguration.java +++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/rm/utils/UserConfiguration.java @@ -119,14 +119,11 @@ public static Resource getUserConfiguredResource( UserCreatorLabel userCreatorLabel, EngineTypeLabel engineTypeLabel) { try { + Map labelTuple = + new HashMap(); + labelTuple.put(userCreatorLabel, engineTypeLabel); Resource userCreatorAvailableResource = - generateResource( - resourceType, - engineMapCache.getCacheMap( - buildRequestLabel( - userCreatorLabel.getUser(), - userCreatorLabel.getCreator(), - engineTypeLabel.getEngineType()))); + generateResource(resourceType, engineMapCache.getCacheMap(labelTuple)); logger.info( userCreatorLabel.getUser() + "on creator " diff --git a/linkis-computation-governance/linkis-manager/linkis-label-common/src/main/java/org/apache/linkis/manager/label/conf/LabelCommonConfig.java b/linkis-computation-governance/linkis-manager/linkis-label-common/src/main/java/org/apache/linkis/manager/label/conf/LabelCommonConfig.java index 89169eb58f..2d180c496e 100644 --- a/linkis-computation-governance/linkis-manager/linkis-label-common/src/main/java/org/apache/linkis/manager/label/conf/LabelCommonConfig.java +++ b/linkis-computation-governance/linkis-manager/linkis-label-common/src/main/java/org/apache/linkis/manager/label/conf/LabelCommonConfig.java @@ -64,7 +64,7 @@ public class LabelCommonConfig { CommonVars.apply("wds.linkis.appconn.engine.version", "1"); public static final CommonVars FLINK_ENGINE_VERSION = - CommonVars.apply("wds.linkis.flink.engine.version", "1.12.2"); + CommonVars.apply("wds.linkis.flink.engine.version", "1.16.2"); public static final CommonVars SQOOP_ENGINE_VERSION = CommonVars.apply("wds.linkis.sqoop.engine.version", "1.4.6"); @@ -75,6 +75,9 @@ public class LabelCommonConfig { public static final CommonVars NEBULA_ENGINE_VERSION = CommonVars.apply("wds.linkis.nebula.engine.version", "3.0.0"); + public static final CommonVars DORIS_ENGINE_VERSION = + CommonVars.apply("linkis.doris.engine.version", "1.2.6"); + public static final CommonVars PRESTO_ENGINE_VERSION = CommonVars.apply("wds.linkis.presto.engine.version", "0.234"); diff --git a/linkis-computation-governance/linkis-manager/linkis-label-common/src/main/java/org/apache/linkis/manager/label/constant/LabelKeyConstant.java b/linkis-computation-governance/linkis-manager/linkis-label-common/src/main/java/org/apache/linkis/manager/label/constant/LabelKeyConstant.java index 8021b35851..fbaea61811 100644 --- a/linkis-computation-governance/linkis-manager/linkis-label-common/src/main/java/org/apache/linkis/manager/label/constant/LabelKeyConstant.java +++ b/linkis-computation-governance/linkis-manager/linkis-label-common/src/main/java/org/apache/linkis/manager/label/constant/LabelKeyConstant.java @@ -66,5 +66,9 @@ public class LabelKeyConstant { public static final String ENGINGE_CONN_RUNTIME_MODE_KEY = "engingeConnRuntimeMode"; + public static final String TEMPLATE_CONF_KEY = "ec.conf.templateId"; + + public static final String TEMPLATE_CONF_NAME_KEY = "ec.resource.name"; + public static final String MANAGER_KEY = "manager"; } diff --git a/linkis-computation-governance/linkis-manager/linkis-label-common/src/main/java/org/apache/linkis/manager/label/entity/engine/EngineType.scala b/linkis-computation-governance/linkis-manager/linkis-label-common/src/main/java/org/apache/linkis/manager/label/entity/engine/EngineType.scala index c5c44017e6..c58f6421be 100644 --- a/linkis-computation-governance/linkis-manager/linkis-label-common/src/main/java/org/apache/linkis/manager/label/entity/engine/EngineType.scala +++ b/linkis-computation-governance/linkis-manager/linkis-label-common/src/main/java/org/apache/linkis/manager/label/entity/engine/EngineType.scala @@ -35,6 +35,8 @@ object EngineType extends Enumeration with Logging { val REPL = Value("repl") + val DORIS = Value("doris") + val SHELL = Value("shell") val JDBC = Value("jdbc") @@ -98,6 +100,7 @@ object EngineType extends Enumeration with Logging { case _ if PRESTO.toString.equalsIgnoreCase(str) => PRESTO case _ if NEBULA.toString.equalsIgnoreCase(str) => NEBULA case _ if REPL.toString.equalsIgnoreCase(str) => REPL + case _ if DORIS.toString.equalsIgnoreCase(str) => DORIS case _ if FLINK.toString.equalsIgnoreCase(str) => FLINK case _ if APPCONN.toString.equals(str) => APPCONN case _ if SQOOP.toString.equalsIgnoreCase(str) => SQOOP diff --git a/linkis-computation-governance/linkis-manager/linkis-label-common/src/main/java/org/apache/linkis/manager/label/entity/engine/RunType.scala b/linkis-computation-governance/linkis-manager/linkis-label-common/src/main/java/org/apache/linkis/manager/label/entity/engine/RunType.scala index 645c15dbdb..df17b6f2d3 100644 --- a/linkis-computation-governance/linkis-manager/linkis-label-common/src/main/java/org/apache/linkis/manager/label/entity/engine/RunType.scala +++ b/linkis-computation-governance/linkis-manager/linkis-label-common/src/main/java/org/apache/linkis/manager/label/entity/engine/RunType.scala @@ -25,6 +25,7 @@ object RunType extends Enumeration { val SCALA = Value("scala") val PYTHON = Value("python") val REPL = Value("repl") + val DORIS = Value("doris") val JAVA = Value("java") val PYSPARK = Value("py") val R = Value("r") diff --git a/linkis-computation-governance/linkis-manager/linkis-label-common/src/main/java/org/apache/linkis/manager/label/utils/EngineTypeLabelCreator.java b/linkis-computation-governance/linkis-manager/linkis-label-common/src/main/java/org/apache/linkis/manager/label/utils/EngineTypeLabelCreator.java index d7911c030e..88cc9139ec 100644 --- a/linkis-computation-governance/linkis-manager/linkis-label-common/src/main/java/org/apache/linkis/manager/label/utils/EngineTypeLabelCreator.java +++ b/linkis-computation-governance/linkis-manager/linkis-label-common/src/main/java/org/apache/linkis/manager/label/utils/EngineTypeLabelCreator.java @@ -76,6 +76,8 @@ private static void init() { EngineType.HBASE().toString(), LabelCommonConfig.HBASE_ENGINE_VERSION.getValue()); defaultVersion.put( EngineType.NEBULA().toString(), LabelCommonConfig.NEBULA_ENGINE_VERSION.getValue()); + defaultVersion.put( + EngineType.DORIS().toString(), LabelCommonConfig.DORIS_ENGINE_VERSION.getValue()); defaultVersion.put( EngineType.SQOOP().toString(), LabelCommonConfig.SQOOP_ENGINE_VERSION.getValue()); defaultVersion.put( diff --git a/linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/java/org/apache/linkis/manager/common/protocol/engine/EngineAskAsyncResponse.java b/linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/java/org/apache/linkis/manager/common/protocol/engine/EngineAskAsyncResponse.java index 364af7fd0a..86fc06698f 100644 --- a/linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/java/org/apache/linkis/manager/common/protocol/engine/EngineAskAsyncResponse.java +++ b/linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/java/org/apache/linkis/manager/common/protocol/engine/EngineAskAsyncResponse.java @@ -45,4 +45,15 @@ public ServiceInstance getManagerInstance() { public void setManagerInstance(ServiceInstance managerInstance) { this.managerInstance = managerInstance; } + + @Override + public String toString() { + return "EngineAskAsyncResponse{" + + "id='" + + id + + '\'' + + ", managerInstance=" + + managerInstance + + '}'; + } } diff --git a/linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/java/org/apache/linkis/manager/common/protocol/engine/EngineConnReleaseRequest.java b/linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/java/org/apache/linkis/manager/common/protocol/engine/EngineConnReleaseRequest.java index 31e269d490..896397033e 100644 --- a/linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/java/org/apache/linkis/manager/common/protocol/engine/EngineConnReleaseRequest.java +++ b/linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/java/org/apache/linkis/manager/common/protocol/engine/EngineConnReleaseRequest.java @@ -82,4 +82,23 @@ public NodeStatus getNodeStatus() { public void setNodeStatus(NodeStatus nodeStatus) { this.nodeStatus = nodeStatus; } + + @Override + public String toString() { + return "EngineConnReleaseRequest{" + + "serviceInstance=" + + serviceInstance + + ", user='" + + user + + '\'' + + ", msg='" + + msg + + '\'' + + ", nodeStatus=" + + nodeStatus + + ", ticketId='" + + ticketId + + '\'' + + '}'; + } } diff --git a/linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/java/org/apache/linkis/manager/common/protocol/engine/EngineConnStatusCallback.java b/linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/java/org/apache/linkis/manager/common/protocol/engine/EngineConnStatusCallback.java index d162257cab..ea0dca16ec 100644 --- a/linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/java/org/apache/linkis/manager/common/protocol/engine/EngineConnStatusCallback.java +++ b/linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/java/org/apache/linkis/manager/common/protocol/engine/EngineConnStatusCallback.java @@ -52,4 +52,17 @@ public NodeStatus getStatus() { public String getInitErrorMsg() { return initErrorMsg; } + + @Override + public String toString() { + return "EngineConnStatusCallback{" + + "serviceInstance=" + + serviceInstance + + ", ticketId='" + + ticketId + + '\'' + + ", status=" + + status + + '}'; + } } diff --git a/linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/java/org/apache/linkis/manager/common/protocol/engine/EngineCreateError.java b/linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/java/org/apache/linkis/manager/common/protocol/engine/EngineCreateError.java index f9c55b1011..0951b5b276 100644 --- a/linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/java/org/apache/linkis/manager/common/protocol/engine/EngineCreateError.java +++ b/linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/java/org/apache/linkis/manager/common/protocol/engine/EngineCreateError.java @@ -58,4 +58,9 @@ public Boolean getRetry() { public void setRetry(Boolean retry) { this.retry = retry; } + + @Override + public String toString() { + return "EngineCreateError{" + "id='" + id + '\'' + ", retry=" + retry + '}'; + } } diff --git a/linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/java/org/apache/linkis/manager/common/protocol/engine/EngineCreateSuccess.java b/linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/java/org/apache/linkis/manager/common/protocol/engine/EngineCreateSuccess.java index 37ae583f88..1b7496c720 100644 --- a/linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/java/org/apache/linkis/manager/common/protocol/engine/EngineCreateSuccess.java +++ b/linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/java/org/apache/linkis/manager/common/protocol/engine/EngineCreateSuccess.java @@ -45,4 +45,9 @@ public EngineNode getEngineNode() { public void setEngineNode(EngineNode engineNode) { this.engineNode = engineNode; } + + @Override + public String toString() { + return "EngineCreateSuccess{" + "id='" + id + '\'' + ", engineNode=" + engineNode + '}'; + } } diff --git a/linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/java/org/apache/linkis/manager/common/protocol/engine/EngineInfoClearRequest.java b/linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/java/org/apache/linkis/manager/common/protocol/engine/EngineInfoClearRequest.java index 1820fe8c94..36aa697284 100644 --- a/linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/java/org/apache/linkis/manager/common/protocol/engine/EngineInfoClearRequest.java +++ b/linkis-computation-governance/linkis-manager/linkis-manager-common/src/main/java/org/apache/linkis/manager/common/protocol/engine/EngineInfoClearRequest.java @@ -43,4 +43,9 @@ public void setUser(String user) { public String getUser() { return user; } + + @Override + public String toString() { + return "EngineInfoClearRequest{" + "engineNode=" + engineNode + ", user='" + user + '\'' + '}'; + } } diff --git a/linkis-dist/bin/checkEnv.sh b/linkis-dist/bin/checkEnv.sh index 68682a2338..049b2e1171 100644 --- a/linkis-dist/bin/checkEnv.sh +++ b/linkis-dist/bin/checkEnv.sh @@ -13,13 +13,13 @@ # See the License for the specific language governing permissions and # limitations under the License. # - +source ~/.bash_profile shellDir=`dirname $0` workDir=`cd ${shellDir}/..;pwd` source ${workDir}/bin/common.sh source ${workDir}/deploy-config/linkis-env.sh source ${workDir}/deploy-config/db.sh - +source ~/.bash_profile say() { printf 'check command fail \n %s\n' "$1" } @@ -169,12 +169,6 @@ echo "check hdfs" need_cmd hdfs echo "check shell" need_cmd $SHELL -echo "check spark-submit" -need_cmd spark-submit -echo "check spark-shell" -need_cmd spark-shell -echo "check spark-sql" -need_cmd spark-sql echo "check hadoop" need_cmd hadoop @@ -187,6 +181,12 @@ checkPythonAndJava checkMysql if [ "$ENABLE_SPARK" == "true" ]; then + echo "check spark-submit" + need_cmd spark-submit + echo "check spark-shell" + need_cmd spark-shell + echo "check spark-sql" + need_cmd spark-sql checkSpark fi @@ -203,8 +203,10 @@ echo -e "\n<-----End to check service status---->" # --- check Service Port echo -e "\n3. <-----Start to check service Port---->" -SERVER_PORT=$EUREKA_PORT -check_service_port +if [ "$DISCOVERY" == "EUREKA" ]; then + SERVER_PORT=$EUREKA_PORT + check_service_port +fi SERVER_PORT=$GATEWAY_PORT check_service_port @@ -227,4 +229,4 @@ if [ "$portIsOccupy" = true ];then exit 1 fi -echo "\n <-----End to check service Port---->" \ No newline at end of file +echo -e "\n<-----End to check service Port---->" diff --git a/linkis-dist/bin/install.sh b/linkis-dist/bin/install.sh index d6e50221d5..24cada6657 100644 --- a/linkis-dist/bin/install.sh +++ b/linkis-dist/bin/install.sh @@ -110,6 +110,15 @@ else isSuccess "cp ${LINKIS_PACKAGE} to $LINKIS_HOME" fi +if [ "$DISCOVERY" == "NACOS" ]; then + rm -rf $LINKIS_HOME/conf/application-* + mv $LINKIS_HOME/conf/nacos/* $LINKIS_HOME/conf + echo "Use NACOS mode" +else + echo "Use EUREKA mode" +fi +rm -rf $LINKIS_HOME/conf/nacos + cp ${LINKIS_CONFIG_PATH} $LINKIS_HOME/conf cp ${LINKIS_DB_CONFIG_PATH} $LINKIS_HOME/conf @@ -407,34 +416,50 @@ fi currentTime=`date +%Y%m%d%H%M%S` -##eureka -sed -i ${txt} "s#defaultZone:.*#defaultZone: $EUREKA_URL#g" $LINKIS_HOME/conf/application-eureka.yml -sed -i ${txt} "s#port:.*#port: $EUREKA_PORT#g" $LINKIS_HOME/conf/application-eureka.yml sed -i ${txt} "s#linkis.app.version:.*#linkis.app.version: $LINKIS_VERSION-$currentTime#g" $LINKIS_HOME/conf/application-eureka.yml - -##server application.yml -sed -i ${txt} "s#defaultZone:.*#defaultZone: $EUREKA_URL#g" $LINKIS_HOME/conf/application-linkis.yml sed -i ${txt} "s#linkis.app.version:.*#linkis.app.version: $LINKIS_VERSION-$currentTime#g" $LINKIS_HOME/conf/application-linkis.yml - -sed -i ${txt} "s#defaultZone:.*#defaultZone: $EUREKA_URL#g" $LINKIS_HOME/conf/application-engineconn.yml sed -i ${txt} "s#linkis.app.version:.*#linkis.app.version: $LINKIS_VERSION-$currentTime#g" $LINKIS_HOME/conf/application-engineconn.yml -if [ "$EUREKA_PREFER_IP" == "true" ]; then - sed -i ${txt} "s/# prefer-ip-address:/prefer-ip-address:/g" $LINKIS_HOME/conf/application-eureka.yml +sed -i ${txt} "s#DISCOVERY=.*#DISCOVERY=$DISCOVERY#g" $LINKIS_HOME/sbin/common.sh + +if [ "$DISCOVERY" == "EUREKA" ]; then + ##eureka + sed -i ${txt} "s#defaultZone:.*#defaultZone: $EUREKA_URL#g" $LINKIS_HOME/conf/application-eureka.yml + sed -i ${txt} "s#port:.*#port: $EUREKA_PORT#g" $LINKIS_HOME/conf/application-eureka.yml + sed -i ${txt} "s#linkis.app.version:.*#linkis.app.version: $LINKIS_VERSION-$currentTime#g" $LINKIS_HOME/conf/application-eureka.yml + + ##server application.yml + sed -i ${txt} "s#defaultZone:.*#defaultZone: $EUREKA_URL#g" $LINKIS_HOME/conf/application-linkis.yml - sed -i ${txt} "s/# prefer-ip-address:/prefer-ip-address:/g" $LINKIS_HOME/conf/application-linkis.yml - sed -i ${txt} "s/# instance-id:/instance-id:/g" $LINKIS_HOME/conf/application-linkis.yml + sed -i ${txt} "s#defaultZone:.*#defaultZone: $EUREKA_URL#g" $LINKIS_HOME/conf/application-engineconn.yml - sed -i ${txt} "s/# prefer-ip-address:/prefer-ip-address:/g" $LINKIS_HOME/conf/application-engineconn.yml - sed -i ${txt} "s/# instance-id:/instance-id:/g" $LINKIS_HOME/conf/application-linkis.yml + if [ "$EUREKA_PREFER_IP" == "true" ]; then + sed -i ${txt} "s/# prefer-ip-address:/prefer-ip-address:/g" $LINKIS_HOME/conf/application-eureka.yml + + sed -i ${txt} "s/# prefer-ip-address:/prefer-ip-address:/g" $LINKIS_HOME/conf/application-linkis.yml + sed -i ${txt} "s/# instance-id:/instance-id:/g" $LINKIS_HOME/conf/application-linkis.yml + + sed -i ${txt} "s/# prefer-ip-address:/prefer-ip-address:/g" $LINKIS_HOME/conf/application-engineconn.yml + sed -i ${txt} "s/# instance-id:/instance-id:/g" $LINKIS_HOME/conf/application-linkis.yml + + sed -i ${txt} "s#linkis.discovery.prefer-ip-address.*#linkis.discovery.prefer-ip-address=true#g" $common_conf + fi + export DISCOVERY_SERVER_ADDRES=$EUREKA_INSTALL_IP:$EUREKA_POR +fi + +if [ "$DISCOVERY" == "NACOS" ]; then + sed -i ${txt} "s#server-addr:.*#server-addr: $NACOS_SERVER_ADDR#g" $LINKIS_HOME/conf/application-linkis.yml + sed -i ${txt} "s#server-addr:.*#server-addr: $NACOS_SERVER_ADDR#g" $LINKIS_HOME/conf/application-engineconn.yml sed -i ${txt} "s#linkis.discovery.prefer-ip-address.*#linkis.discovery.prefer-ip-address=true#g" $common_conf + + export DISCOVERY_SERVER_ADDRES=$NACOS_SERVER_ADDR fi echo "update conf $common_conf" sed -i ${txt} "s#wds.linkis.server.version.*#wds.linkis.server.version=$LINKIS_SERVER_VERSION#g" $common_conf sed -i ${txt} "s#wds.linkis.gateway.url.*#wds.linkis.gateway.url=http://$GATEWAY_INSTALL_IP:$GATEWAY_PORT#g" $common_conf -sed -i ${txt} "s#linkis.discovery.server-address.*#linkis.discovery.server-address=http://$EUREKA_INSTALL_IP:$EUREKA_PORT#g" $common_conf +sed -i ${txt} "s#linkis.discovery.server-address.*#linkis.discovery.server-address=http://$DISCOVERY_SERVER_ADDRES#g" $common_conf if [[ 'postgresql' = "$dbType" ]];then sed -i ${txt} "s#wds.linkis.server.mybatis.datasource.url.*#wds.linkis.server.mybatis.datasource.url=jdbc:postgresql://${PG_HOST}:${PG_PORT}/${PG_DB}?currentSchema=${PG_SCHEMA}\&stringtype=unspecified#g" $common_conf sed -i ${txt} "s#wds.linkis.server.mybatis.datasource.username.*#wds.linkis.server.mybatis.datasource.username=$PG_USER#g" $common_conf diff --git a/linkis-dist/deploy-config/linkis-env.sh b/linkis-dist/deploy-config/linkis-env.sh index f37d0720a8..5578e25277 100644 --- a/linkis-dist/deploy-config/linkis-env.sh +++ b/linkis-dist/deploy-config/linkis-env.sh @@ -111,6 +111,9 @@ SPARK_CONF_DIR=/appcom/config/spark-config # Linkis in a distributed manner and set the following microservice parameters # +### DISCOVERY +DISCOVERY=EUREKA + ### EUREKA install information ### You can access it in your browser at the address below:http://${EUREKA_INSTALL_IP}:${EUREKA_PORT} #EUREKA: Microservices Service Registration Discovery Center @@ -119,6 +122,10 @@ EUREKA_PORT=20303 export EUREKA_PREFER_IP=false #EUREKA_HEAP_SIZE="512M" +### NACOS install information +### NACOS +NACOS_SERVER_ADDR=127.0.0.1:8848 + ##linkis-mg-gateway #GATEWAY_INSTALL_IP=127.0.0.1 GATEWAY_PORT=9001 @@ -160,7 +167,7 @@ export SERVER_HEAP_SIZE="512M" ##The extended lib such mysql-connector-java-*.jar #LINKIS_EXTENDED_LIB=/appcom/common/linkisExtendedLib -LINKIS_VERSION=1.4.0 +LINKIS_VERSION=1.5.0 # for install LINKIS_PUBLIC_MODULE=lib/linkis-commons/public-module diff --git a/linkis-dist/package/conf/nacos/application-engineconn.yml b/linkis-dist/package/conf/nacos/application-engineconn.yml new file mode 100644 index 0000000000..0cf4dd90b9 --- /dev/null +++ b/linkis-dist/package/conf/nacos/application-engineconn.yml @@ -0,0 +1,38 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +spring: + application: + name: linkis-cg-engineconn + servlet: + multipart: + max-file-size: 500MB + max-request-size: 500MB + file-size-threshold: 50MB + cloud: + nacos: + discovery: + server-addr: 127.0.0.1:8848 + metadata: + prometheus.path: ${prometheus.path:/api/rest_j/v1/actuator/prometheus} + linkis.app.version: ${linkis.app.version} + +management: + endpoints: + web: + exposure: + include: refresh,info,health,metrics + +logging: + config: classpath:log4j2.xml diff --git a/linkis-dist/package/conf/nacos/application-linkis.yml b/linkis-dist/package/conf/nacos/application-linkis.yml new file mode 100644 index 0000000000..07dc8adeb3 --- /dev/null +++ b/linkis-dist/package/conf/nacos/application-linkis.yml @@ -0,0 +1,59 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +nacos: + discovery: + server-addr: 127.0.0.1:8848 + metadata: + prometheus.path: ${prometheus.path:/api/rest_j/v1/actuator/prometheus} + linkis.app.version: ${linkis.app.version} + +management: + endpoints: + web: + exposure: + include: refresh,info,health,metrics + +logging: + config: classpath:log4j2.xml + +pagehelper: + helper-dialect: mysql + reasonable: true + support-methods-arguments: true + params: countSql + +spring: + servlet: + multipart: + max-file-size: 500MB + max-request-size: 500MB + file-size-threshold: 50MB +#ribbon: +# ReadTimeout: 10000 +# ConnectTimeout: 10000 + cloud: + nacos: + discovery: + server-addr: 127.0.0.1:8848 + metadata: + prometheus.path: ${prometheus.path:/api/rest_j/v1/actuator/prometheus} + linkis.app.version: ${linkis.app.version} + + +##disable kinif4j.production when you want to use apidoc during development +knife4j: + enable: true + production: true diff --git a/linkis-dist/package/db/linkis_ddl.sql b/linkis-dist/package/db/linkis_ddl.sql index f7f8c3547b..052da64168 100644 --- a/linkis-dist/package/db/linkis_ddl.sql +++ b/linkis-dist/package/db/linkis_ddl.sql @@ -46,6 +46,7 @@ CREATE TABLE `linkis_ps_configuration_config_key`( `en_description` varchar(200) DEFAULT NULL COMMENT 'english description', `en_name` varchar(100) DEFAULT NULL COMMENT 'english name', `en_treeName` varchar(100) DEFAULT NULL COMMENT 'english treeName', + `template_required` tinyint(1) DEFAULT 0 COMMENT 'template required 0 none / 1 must', PRIMARY KEY (`id`) )ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_bin; @@ -64,7 +65,7 @@ DROP TABLE IF EXISTS `linkis_ps_configuration_config_value`; CREATE TABLE `linkis_ps_configuration_config_value`( `id` bigint(20) NOT NULL AUTO_INCREMENT, `config_key_id` bigint(20), - `config_value` varchar(200), + `config_value` varchar(500), `config_label_id`int(20), `update_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP, `create_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP, @@ -85,7 +86,25 @@ CREATE TABLE `linkis_ps_configuration_category` ( UNIQUE INDEX `uniq_label_id` (`label_id`) ) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_bin; - +DROP TABLE IF EXISTS `linkis_ps_configuration_template_config_key`; +CREATE TABLE IF NOT EXISTS `linkis_ps_configuration_template_config_key` ( + `id` BIGINT(20) NOT NULL AUTO_INCREMENT, + `template_name` VARCHAR(200) NOT NULL COMMENT 'Configuration template name redundant storage', + `template_uuid` VARCHAR(36) NOT NULL COMMENT 'uuid template id recorded by the third party', + `key_id` BIGINT(20) NOT NULL COMMENT 'id of linkis_ps_configuration_config_key', + `config_value` VARCHAR(200) NULL DEFAULT NULL COMMENT 'configuration value', + `max_value` VARCHAR(50) NULL DEFAULT NULL COMMENT 'upper limit value', + `min_value` VARCHAR(50) NULL DEFAULT NULL COMMENT 'Lower limit value (reserved)', + `validate_range` VARCHAR(50) NULL DEFAULT NULL COMMENT 'Verification regularity (reserved)', + `is_valid` VARCHAR(2) DEFAULT 'Y' COMMENT 'Is it valid? Reserved Y/N', + `create_by` VARCHAR(50) NOT NULL COMMENT 'Creator', + `create_time` DATETIME DEFAULT CURRENT_TIMESTAMP COMMENT 'create time', + `update_by` VARCHAR(50) NULL DEFAULT NULL COMMENT 'Update by', + `update_time` DATETIME DEFAULT CURRENT_TIMESTAMP COMMENT 'update time', + PRIMARY KEY (`id`), + UNIQUE INDEX `uniq_tid_kid` (`template_uuid`, `key_id`), + UNIQUE INDEX `uniq_tname_kid` (`template_uuid`, `key_id`) + )ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin; DROP TABLE IF EXISTS `linkis_ps_configuration_key_limit_for_user`; CREATE TABLE IF NOT EXISTS `linkis_ps_configuration_key_limit_for_user` ( diff --git a/linkis-dist/package/db/linkis_ddl_pg.sql b/linkis-dist/package/db/linkis_ddl_pg.sql index 8d66c0dd8e..ef3eeec34c 100644 --- a/linkis-dist/package/db/linkis_ddl_pg.sql +++ b/linkis-dist/package/db/linkis_ddl_pg.sql @@ -40,12 +40,12 @@ CREATE TABLE linkis_ps_configuration_config_key ( is_hidden bool NULL, is_advanced bool NULL, "level" int2 NULL, - boundary_type int2 null, - "treeName" varchar(20) NULL, + "treeName" varchar(20) NULL, + boundary_type int2 NOT NULL DEFAULT 0, en_description varchar(200) NULL, en_name varchar(100) NULL, "en_treeName" varchar(100) NULL, - `boundary_type` int2 NOT NULL, + template_required bool NULL, CONSTRAINT linkis_configuration_config_key_pkey PRIMARY KEY (id) ); COMMENT ON COLUMN "linkis_ps_configuration_config_key"."key" IS 'Set key, e.g. spark.executor.instances'; @@ -136,7 +136,7 @@ CREATE TABLE IF NOT EXISTS linkis_ps_configuration_key_limit_for_user ( DROP TABLE IF EXISTS linkis_ps_configutation_lm_across_cluster_rule; CREATE TABLE IF NOT EXISTS linkis_ps_configutation_lm_across_cluster_rule ( - rule_id INT PRIMARY KEY AUTOINCREMENT, + rule_id bigserial NOT NULL, cluster_name varchar(32) NOT NULL, creator varchar(32) NOT NULL, username varchar(32) NOT NULL, diff --git a/linkis-dist/package/db/linkis_dml.sql b/linkis-dist/package/db/linkis_dml.sql index 8a1dcffa3a..b58197f38d 100644 --- a/linkis-dist/package/db/linkis_dml.sql +++ b/linkis-dist/package/db/linkis_dml.sql @@ -777,3 +777,6 @@ VALUES (@data_source_type_id, 'address', '地址', 'Address', NULL, 'TEXT', NULL (@data_source_type_id, 'username', '用户名(Username)', 'Username', NULL, 'TEXT', NULL, 1, '用户名(Username)', 'Username', '^[0-9A-Za-z_-]+$', NULL, NULL, NULL, now(), now()), (@data_source_type_id, 'password', '密码(Password)', 'Password', NULL, 'PASSWORD', NULL, 1, '密码(Password)', 'Password', '', NULL, NULL, NULL, now(), now()), (@data_source_type_id, 'instance', '实例名(instance)', 'Instance', NULL, 'TEXT', NULL, 1, '实例名(instance)', 'Instance', NULL, NULL, NULL, NULL, now(), now()); + +select @data_source_type_id := id from `linkis_ps_dm_datasource_type` where `name` = 'doris'; +UPDATE linkis_ps_dm_datasource_type_key SET `require` = 0 WHERE `key` ="password" and `data_source_type_id` = @data_source_type_id; diff --git a/linkis-dist/package/db/linkis_dml_pg.sql b/linkis-dist/package/db/linkis_dml_pg.sql index ad1064aa3d..6139f1e6af 100644 --- a/linkis-dist/package/db/linkis_dml_pg.sql +++ b/linkis-dist/package/db/linkis_dml_pg.sql @@ -647,7 +647,7 @@ VALUES ((select id from "linkis_ps_dm_datasource_type" where "name" = 'doris'), ((select id from "linkis_ps_dm_datasource_type" where "name" = 'doris'), 'driverClassName', '驱动类名(Driver class name)', 'Driver class name', 'com.mysql.jdbc.Driver', 'TEXT', NULL, '1', '驱动类名(Driver class name)', 'Driver class name', NULL, NULL, NULL, NULL, now(), now()), ((select id from "linkis_ps_dm_datasource_type" where "name" = 'doris'), 'params', '连接参数(Connection params)', 'Connection params', NULL, 'TEXT', NULL, '0', '输入JSON格式(Input JSON format): {"param":"value"}', 'Input JSON format: {"param":"value"}', NULL, NULL, NULL, NULL, now(), now()), ((select id from "linkis_ps_dm_datasource_type" where "name" = 'doris'), 'username', '用户名(Username)', 'Username', NULL, 'TEXT', NULL, '1', '用户名(Username)', 'Username', '^[0-9A-Za-z_-]+$', NULL, NULL, NULL, now(), now()), - ((select id from "linkis_ps_dm_datasource_type" where "name" = 'doris'), 'password', '密码(Password)', 'Password', NULL, 'PASSWORD', NULL, '1', '密码(Password)', 'Password', '', NULL, NULL, NULL, now(), now()), + ((select id from "linkis_ps_dm_datasource_type" where "name" = 'doris'), 'password', '密码(Password)', 'Password', NULL, 'PASSWORD', NULL, '0', '密码(Password)', 'Password', '', NULL, NULL, NULL, now(), now()), ((select id from "linkis_ps_dm_datasource_type" where "name" = 'doris'), 'instance', '实例名(instance)', 'Instance', NULL, 'TEXT', NULL, '1', '实例名(instance)', 'Instance', NULL, NULL, NULL, NULL, now(), now()); -- https://github.com/ClickHouse/clickhouse-jdbc/tree/master/clickhouse-jdbc diff --git a/linkis-dist/package/db/upgrade/1.6.0_schema/mysql/linkis_dml.sql b/linkis-dist/package/db/upgrade/1.6.0_schema/mysql/linkis_dml.sql new file mode 100644 index 0000000000..0c9b591a27 --- /dev/null +++ b/linkis-dist/package/db/upgrade/1.6.0_schema/mysql/linkis_dml.sql @@ -0,0 +1,19 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +select @data_source_type_id := id from `linkis_ps_dm_datasource_type` where `name` = 'doris'; +UPDATE linkis_ps_dm_datasource_type_key SET `require` = 0 WHERE `key` ="password" and `data_source_type_id` = @data_source_type_id; diff --git a/linkis-dist/package/sbin/common.sh b/linkis-dist/package/sbin/common.sh index 6a9acb207f..f3f0555933 100644 --- a/linkis-dist/package/sbin/common.sh +++ b/linkis-dist/package/sbin/common.sh @@ -27,6 +27,8 @@ NC='\033[0m' # No Color GREEN='\033[0;32m' #used as: echo -e "Apache ${RED}Linkis ${NC} Test \n" +export DISCOVERY=EUREKA + function isLocal(){ if [ "$1" == "127.0.0.1" ];then return 0 diff --git a/linkis-dist/package/sbin/linkis-start-all.sh b/linkis-dist/package/sbin/linkis-start-all.sh index c9bd380619..d70b37d19d 100644 --- a/linkis-dist/package/sbin/linkis-start-all.sh +++ b/linkis-dist/package/sbin/linkis-start-all.sh @@ -64,10 +64,12 @@ echo "<-------------------------------->" echo "Linkis manager data is being cleared" sh $LINKIS_HOME/sbin/clear-server.sh -#linkis-mg-eureka -export SERVER_NAME="mg-eureka" -SERVER_IP=$EUREKA_INSTALL_IP -startApp +if [ "$DISCOVERY" == "EUREKA" ]; then + #linkis-mg-eureka + export SERVER_NAME="mg-eureka" + SERVER_IP=$EUREKA_INSTALL_IP + startApp +fi #linkis-mg-gateway SERVER_NAME="mg-gateway" @@ -131,9 +133,11 @@ sleep 3 } #linkis-mg-eureka -export SERVER_NAME="mg-eureka" -SERVER_IP=$EUREKA_INSTALL_IP -checkServer +if [ "$DISCOVERY" == "EUREKA" ]; then + export SERVER_NAME="mg-eureka" + SERVER_IP=$EUREKA_INSTALL_IP + checkServer +fi #linkis-mg-gateway diff --git a/linkis-dist/package/sbin/linkis-stop-all.sh b/linkis-dist/package/sbin/linkis-stop-all.sh index a560075a95..4302cf7af1 100644 --- a/linkis-dist/package/sbin/linkis-stop-all.sh +++ b/linkis-dist/package/sbin/linkis-stop-all.sh @@ -103,8 +103,10 @@ SERVER_IP=$MANAGER_INSTALL_IP stopApp #linkis-mg-eureka -export SERVER_NAME="mg-eureka" -SERVER_IP=$EUREKA_INSTALL_IP -stopApp +if [ "$DISCOVERY" == "EUREKA" ]; then + export SERVER_NAME="mg-eureka" + SERVER_IP=$EUREKA_INSTALL_IP + stopApp +fi echo "stop-all shell script executed completely" diff --git a/linkis-dist/release-docs/licenses/LICENSE-client.txt b/linkis-dist/release-docs/licenses/LICENSE-client.txt new file mode 100644 index 0000000000..d645695673 --- /dev/null +++ b/linkis-dist/release-docs/licenses/LICENSE-client.txt @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/linkis-dist/release-docs/licenses/LICENSE-jts-core.txt b/linkis-dist/release-docs/licenses/LICENSE-jts-core.txt new file mode 100644 index 0000000000..e55f34467e --- /dev/null +++ b/linkis-dist/release-docs/licenses/LICENSE-jts-core.txt @@ -0,0 +1,277 @@ +Eclipse Public License - v 2.0 + + THE ACCOMPANYING PROGRAM IS PROVIDED UNDER THE TERMS OF THIS ECLIPSE + PUBLIC LICENSE ("AGREEMENT"). ANY USE, REPRODUCTION OR DISTRIBUTION + OF THE PROGRAM CONSTITUTES RECIPIENT'S ACCEPTANCE OF THIS AGREEMENT. + +1. DEFINITIONS + +"Contribution" means: + + a) in the case of the initial Contributor, the initial content + Distributed under this Agreement, and + + b) in the case of each subsequent Contributor: + i) changes to the Program, and + ii) additions to the Program; + where such changes and/or additions to the Program originate from + and are Distributed by that particular Contributor. A Contribution + "originates" from a Contributor if it was added to the Program by + such Contributor itself or anyone acting on such Contributor's behalf. + Contributions do not include changes or additions to the Program that + are not Modified Works. + +"Contributor" means any person or entity that Distributes the Program. + +"Licensed Patents" mean patent claims licensable by a Contributor which +are necessarily infringed by the use or sale of its Contribution alone +or when combined with the Program. + +"Program" means the Contributions Distributed in accordance with this +Agreement. + +"Recipient" means anyone who receives the Program under this Agreement +or any Secondary License (as applicable), including Contributors. + +"Derivative Works" shall mean any work, whether in Source Code or other +form, that is based on (or derived from) the Program and for which the +editorial revisions, annotations, elaborations, or other modifications +represent, as a whole, an original work of authorship. + +"Modified Works" shall mean any work in Source Code or other form that +results from an addition to, deletion from, or modification of the +contents of the Program, including, for purposes of clarity any new file +in Source Code form that contains any contents of the Program. Modified +Works shall not include works that contain only declarations, +interfaces, types, classes, structures, or files of the Program solely +in each case in order to link to, bind by name, or subclass the Program +or Modified Works thereof. + +"Distribute" means the acts of a) distributing or b) making available +in any manner that enables the transfer of a copy. + +"Source Code" means the form of a Program preferred for making +modifications, including but not limited to software source code, +documentation source, and configuration files. + +"Secondary License" means either the GNU General Public License, +Version 2.0, or any later versions of that license, including any +exceptions or additional permissions as identified by the initial +Contributor. + +2. GRANT OF RIGHTS + + a) Subject to the terms of this Agreement, each Contributor hereby + grants Recipient a non-exclusive, worldwide, royalty-free copyright + license to reproduce, prepare Derivative Works of, publicly display, + publicly perform, Distribute and sublicense the Contribution of such + Contributor, if any, and such Derivative Works. + + b) Subject to the terms of this Agreement, each Contributor hereby + grants Recipient a non-exclusive, worldwide, royalty-free patent + license under Licensed Patents to make, use, sell, offer to sell, + import and otherwise transfer the Contribution of such Contributor, + if any, in Source Code or other form. This patent license shall + apply to the combination of the Contribution and the Program if, at + the time the Contribution is added by the Contributor, such addition + of the Contribution causes such combination to be covered by the + Licensed Patents. The patent license shall not apply to any other + combinations which include the Contribution. No hardware per se is + licensed hereunder. + + c) Recipient understands that although each Contributor grants the + licenses to its Contributions set forth herein, no assurances are + provided by any Contributor that the Program does not infringe the + patent or other intellectual property rights of any other entity. + Each Contributor disclaims any liability to Recipient for claims + brought by any other entity based on infringement of intellectual + property rights or otherwise. As a condition to exercising the + rights and licenses granted hereunder, each Recipient hereby + assumes sole responsibility to secure any other intellectual + property rights needed, if any. For example, if a third party + patent license is required to allow Recipient to Distribute the + Program, it is Recipient's responsibility to acquire that license + before distributing the Program. + + d) Each Contributor represents that to its knowledge it has + sufficient copyright rights in its Contribution, if any, to grant + the copyright license set forth in this Agreement. + + e) Notwithstanding the terms of any Secondary License, no + Contributor makes additional grants to any Recipient (other than + those set forth in this Agreement) as a result of such Recipient's + receipt of the Program under the terms of a Secondary License + (if permitted under the terms of Section 3). + +3. REQUIREMENTS + +3.1 If a Contributor Distributes the Program in any form, then: + + a) the Program must also be made available as Source Code, in + accordance with section 3.2, and the Contributor must accompany + the Program with a statement that the Source Code for the Program + is available under this Agreement, and informs Recipients how to + obtain it in a reasonable manner on or through a medium customarily + used for software exchange; and + + b) the Contributor may Distribute the Program under a license + different than this Agreement, provided that such license: + i) effectively disclaims on behalf of all other Contributors all + warranties and conditions, express and implied, including + warranties or conditions of title and non-infringement, and + implied warranties or conditions of merchantability and fitness + for a particular purpose; + + ii) effectively excludes on behalf of all other Contributors all + liability for damages, including direct, indirect, special, + incidental and consequential damages, such as lost profits; + + iii) does not attempt to limit or alter the recipients' rights + in the Source Code under section 3.2; and + + iv) requires any subsequent distribution of the Program by any + party to be under a license that satisfies the requirements + of this section 3. + +3.2 When the Program is Distributed as Source Code: + + a) it must be made available under this Agreement, or if the + Program (i) is combined with other material in a separate file or + files made available under a Secondary License, and (ii) the initial + Contributor attached to the Source Code the notice described in + Exhibit A of this Agreement, then the Program may be made available + under the terms of such Secondary Licenses, and + + b) a copy of this Agreement must be included with each copy of + the Program. + +3.3 Contributors may not remove or alter any copyright, patent, +trademark, attribution notices, disclaimers of warranty, or limitations +of liability ("notices") contained within the Program from any copy of +the Program which they Distribute, provided that Contributors may add +their own appropriate notices. + +4. COMMERCIAL DISTRIBUTION + +Commercial distributors of software may accept certain responsibilities +with respect to end users, business partners and the like. While this +license is intended to facilitate the commercial use of the Program, +the Contributor who includes the Program in a commercial product +offering should do so in a manner which does not create potential +liability for other Contributors. Therefore, if a Contributor includes +the Program in a commercial product offering, such Contributor +("Commercial Contributor") hereby agrees to defend and indemnify every +other Contributor ("Indemnified Contributor") against any losses, +damages and costs (collectively "Losses") arising from claims, lawsuits +and other legal actions brought by a third party against the Indemnified +Contributor to the extent caused by the acts or omissions of such +Commercial Contributor in connection with its distribution of the Program +in a commercial product offering. The obligations in this section do not +apply to any claims or Losses relating to any actual or alleged +intellectual property infringement. In order to qualify, an Indemnified +Contributor must: a) promptly notify the Commercial Contributor in +writing of such claim, and b) allow the Commercial Contributor to control, +and cooperate with the Commercial Contributor in, the defense and any +related settlement negotiations. The Indemnified Contributor may +participate in any such claim at its own expense. + +For example, a Contributor might include the Program in a commercial +product offering, Product X. That Contributor is then a Commercial +Contributor. If that Commercial Contributor then makes performance +claims, or offers warranties related to Product X, those performance +claims and warranties are such Commercial Contributor's responsibility +alone. Under this section, the Commercial Contributor would have to +defend claims against the other Contributors related to those performance +claims and warranties, and if a court requires any other Contributor to +pay any damages as a result, the Commercial Contributor must pay +those damages. + +5. NO WARRANTY + +EXCEPT AS EXPRESSLY SET FORTH IN THIS AGREEMENT, AND TO THE EXTENT +PERMITTED BY APPLICABLE LAW, THE PROGRAM IS PROVIDED ON AN "AS IS" +BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, EITHER EXPRESS OR +IMPLIED INCLUDING, WITHOUT LIMITATION, ANY WARRANTIES OR CONDITIONS OF +TITLE, NON-INFRINGEMENT, MERCHANTABILITY OR FITNESS FOR A PARTICULAR +PURPOSE. Each Recipient is solely responsible for determining the +appropriateness of using and distributing the Program and assumes all +risks associated with its exercise of rights under this Agreement, +including but not limited to the risks and costs of program errors, +compliance with applicable laws, damage to or loss of data, programs +or equipment, and unavailability or interruption of operations. + +6. DISCLAIMER OF LIABILITY + +EXCEPT AS EXPRESSLY SET FORTH IN THIS AGREEMENT, AND TO THE EXTENT +PERMITTED BY APPLICABLE LAW, NEITHER RECIPIENT NOR ANY CONTRIBUTORS +SHALL HAVE ANY LIABILITY FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING WITHOUT LIMITATION LOST +PROFITS), HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OR DISTRIBUTION OF THE PROGRAM OR THE +EXERCISE OF ANY RIGHTS GRANTED HEREUNDER, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGES. + +7. GENERAL + +If any provision of this Agreement is invalid or unenforceable under +applicable law, it shall not affect the validity or enforceability of +the remainder of the terms of this Agreement, and without further +action by the parties hereto, such provision shall be reformed to the +minimum extent necessary to make such provision valid and enforceable. + +If Recipient institutes patent litigation against any entity +(including a cross-claim or counterclaim in a lawsuit) alleging that the +Program itself (excluding combinations of the Program with other software +or hardware) infringes such Recipient's patent(s), then such Recipient's +rights granted under Section 2(b) shall terminate as of the date such +litigation is filed. + +All Recipient's rights under this Agreement shall terminate if it +fails to comply with any of the material terms or conditions of this +Agreement and does not cure such failure in a reasonable period of +time after becoming aware of such noncompliance. If all Recipient's +rights under this Agreement terminate, Recipient agrees to cease use +and distribution of the Program as soon as reasonably practicable. +However, Recipient's obligations under this Agreement and any licenses +granted by Recipient relating to the Program shall continue and survive. + +Everyone is permitted to copy and distribute copies of this Agreement, +but in order to avoid inconsistency the Agreement is copyrighted and +may only be modified in the following manner. The Agreement Steward +reserves the right to publish new versions (including revisions) of +this Agreement from time to time. No one other than the Agreement +Steward has the right to modify this Agreement. The Eclipse Foundation +is the initial Agreement Steward. The Eclipse Foundation may assign the +responsibility to serve as the Agreement Steward to a suitable separate +entity. Each new version of the Agreement will be given a distinguishing +version number. The Program (including Contributions) may always be +Distributed subject to the version of the Agreement under which it was +received. In addition, after a new version of the Agreement is published, +Contributor may elect to Distribute the Program (including its +Contributions) under the new version. + +Except as expressly stated in Sections 2(a) and 2(b) above, Recipient +receives no rights or licenses to the intellectual property of any +Contributor under this Agreement, whether expressly, by implication, +estoppel or otherwise. All rights in the Program not expressly granted +under this Agreement are reserved. Nothing in this Agreement is intended +to be enforceable by any entity that is not a Contributor or Recipient. +No third-party beneficiary rights are created under this Agreement. + +Exhibit A - Form of Secondary Licenses Notice + +"This Source Code may also be made available under the following +Secondary Licenses when the conditions for such availability set forth +in the Eclipse Public License, v. 2.0 are satisfied: {name license(s), +version(s), and exceptions or additional permissions here}." + + Simply including a copy of this Agreement, including this Exhibit A + is not sufficient to license the Source Code under Secondary Licenses. + + If it is not possible or desirable to put the notice in a particular + file, then You may include the notice in a location (such as a LICENSE + file in a relevant directory) where a recipient would be likely to + look for such a notice. + + You may add additional accurate notices of copyright ownership. \ No newline at end of file diff --git a/linkis-engineconn-plugins/doris/pom.xml b/linkis-engineconn-plugins/doris/pom.xml new file mode 100644 index 0000000000..c108c9c3ea --- /dev/null +++ b/linkis-engineconn-plugins/doris/pom.xml @@ -0,0 +1,110 @@ + + + + 4.0.0 + + org.apache.linkis + linkis + ${revision} + ../../pom.xml + + + linkis-engineplugin-doris + + + + org.apache.linkis + linkis-engineconn-plugin-core + ${project.version} + + + + org.apache.linkis + linkis-computation-engineconn + ${project.version} + + + + org.apache.linkis + linkis-storage + ${project.version} + provided + + + + org.apache.linkis + linkis-rpc + ${project.version} + provided + + + + org.apache.linkis + linkis-common + ${project.version} + provided + + + + + org.apache.httpcomponents + httpclient + ${httpclient.version} + + + + + + + + net.alchim31.maven + scala-maven-plugin + + + + org.apache.maven.plugins + maven-assembly-plugin + false + + false + out + false + false + + src/main/assembly/distribution.xml + + + + + make-assembly + + single + + package + + + src/main/assembly/distribution.xml + + + + + + + + + diff --git a/linkis-engineconn-plugins/doris/src/main/assembly/distribution.xml b/linkis-engineconn-plugins/doris/src/main/assembly/distribution.xml new file mode 100644 index 0000000000..37dac510e4 --- /dev/null +++ b/linkis-engineconn-plugins/doris/src/main/assembly/distribution.xml @@ -0,0 +1,71 @@ + + + + + linkis-engineplugin-doris + + dir + zip + + true + doris + + + + + + /dist/${doris.version}/lib + true + true + false + false + true + + + + + + + + ${basedir}/src/main/resources + + linkis-engineconn.properties + log4j2.xml + + 0777 + dist/${doris.version}/conf + unix + + + + ${basedir}/target + + *.jar + + + *doc.jar + + 0777 + plugin/${doris.version} + + + + + + diff --git a/linkis-engineconn-plugins/doris/src/main/java/org/apache/linkis/engineplugin/doris/DorisEngineConnPlugin.java b/linkis-engineconn-plugins/doris/src/main/java/org/apache/linkis/engineplugin/doris/DorisEngineConnPlugin.java new file mode 100644 index 0000000000..68864e13a3 --- /dev/null +++ b/linkis-engineconn-plugins/doris/src/main/java/org/apache/linkis/engineplugin/doris/DorisEngineConnPlugin.java @@ -0,0 +1,72 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.engineplugin.doris; + +import org.apache.linkis.engineplugin.doris.builder.DorisProcessEngineConnLaunchBuilder; +import org.apache.linkis.engineplugin.doris.factory.DorisEngineConnFactory; +import org.apache.linkis.manager.engineplugin.common.EngineConnPlugin; +import org.apache.linkis.manager.engineplugin.common.creation.EngineConnFactory; +import org.apache.linkis.manager.engineplugin.common.launch.EngineConnLaunchBuilder; +import org.apache.linkis.manager.engineplugin.common.resource.EngineResourceFactory; +import org.apache.linkis.manager.engineplugin.common.resource.GenericEngineResourceFactory; +import org.apache.linkis.manager.label.entity.Label; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +public class DorisEngineConnPlugin implements EngineConnPlugin { + private Object resourceLocker = new Object(); + private Object engineFactoryLocker = new Object(); + private volatile EngineResourceFactory engineResourceFactory; + private volatile EngineConnFactory engineFactory; + private List> defaultLabels = new ArrayList<>(); + + @Override + public void init(Map params) {} + + @Override + public EngineResourceFactory getEngineResourceFactory() { + if (null == engineResourceFactory) { + synchronized (resourceLocker) { + engineResourceFactory = new GenericEngineResourceFactory(); + } + } + return engineResourceFactory; + } + + @Override + public EngineConnLaunchBuilder getEngineConnLaunchBuilder() { + return new DorisProcessEngineConnLaunchBuilder(); + } + + @Override + public EngineConnFactory getEngineConnFactory() { + if (null == engineFactory) { + synchronized (engineFactoryLocker) { + engineFactory = new DorisEngineConnFactory(); + } + } + return engineFactory; + } + + @Override + public List> getDefaultLabels() { + return defaultLabels; + } +} diff --git a/linkis-engineconn-plugins/doris/src/main/java/org/apache/linkis/engineplugin/doris/builder/DorisProcessEngineConnLaunchBuilder.java b/linkis-engineconn-plugins/doris/src/main/java/org/apache/linkis/engineplugin/doris/builder/DorisProcessEngineConnLaunchBuilder.java new file mode 100644 index 0000000000..73c9f6458a --- /dev/null +++ b/linkis-engineconn-plugins/doris/src/main/java/org/apache/linkis/engineplugin/doris/builder/DorisProcessEngineConnLaunchBuilder.java @@ -0,0 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.engineplugin.doris.builder; + +import org.apache.linkis.manager.engineplugin.common.launch.process.JavaProcessEngineConnLaunchBuilder; + +public class DorisProcessEngineConnLaunchBuilder extends JavaProcessEngineConnLaunchBuilder {} diff --git a/linkis-engineconn-plugins/doris/src/main/java/org/apache/linkis/engineplugin/doris/conf/DorisConfiguration.java b/linkis-engineconn-plugins/doris/src/main/java/org/apache/linkis/engineplugin/doris/conf/DorisConfiguration.java new file mode 100644 index 0000000000..4d691a9c30 --- /dev/null +++ b/linkis-engineconn-plugins/doris/src/main/java/org/apache/linkis/engineplugin/doris/conf/DorisConfiguration.java @@ -0,0 +1,89 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.engineplugin.doris.conf; + +import org.apache.linkis.common.conf.CommonVars; + +public class DorisConfiguration { + + public static final CommonVars ENGINE_CONCURRENT_LIMIT = + CommonVars.apply("linkis.engineconn.concurrent.limit", 100); + + public static final CommonVars ENGINE_DEFAULT_LIMIT = + CommonVars.apply("linkis.ec.doris.default.limit", 5000); + + public static final CommonVars DORIS_COLUMN_SEPARATOR = + CommonVars.apply("linkis.ec.doris.column.separator", ","); + + public static final CommonVars DORIS_LINE_DELIMITER = + CommonVars.apply("linkis.ec.doris.line.delimiter", "\\n"); + + public static final CommonVars DORIS_STREAM_LOAD_FILE_PATH = + CommonVars.apply( + "linkis.ec.doris.stream.load.file.path", + "", + "A file path, for example: /test/test.csv, currently supports csv、json、parquet、orc format"); + + public static final CommonVars DORIS_COLUMNS = + CommonVars.apply("linkis.ec.doris.columns", ""); + + public static final CommonVars DORIS_LABEL = + CommonVars.apply("linkis.ec.doris.label", ""); + + public static final CommonVars DORIS_CONF = + CommonVars.apply( + "linkis.ec.doris.conf", + "", + "The doris parameter, separated by commas, for example: timeout:600,label:123"); + + public static final CommonVars DORIS_HOST = + CommonVars.apply("linkis.ec.doris.host", "127.0.0.1"); + + public static final CommonVars DORIS_HTTP_PORT = + CommonVars.apply("linkis.ec.doris.http.port", 8030); + + public static final CommonVars DORIS_JDBC_PORT = + CommonVars.apply("linkis.ec.doris.jdcb.port", 9030); + + public static final CommonVars DORIS_DATABASE = + CommonVars.apply("linkis.ec.doris.database", ""); + + public static final CommonVars DORIS_TABLE = + CommonVars.apply("linkis.ec.doris.table", ""); + + public static final CommonVars DORIS_USER_NAME = + CommonVars.apply("linkis.ec.doris.username", "root"); + + public static final CommonVars DORIS_PASSWORD = + CommonVars.apply("linkis.ec.doris.password", ""); + + public static final CommonVars DORIS_DATASOURCE = + CommonVars.apply("linkis.ec.doris.datasource", ""); + + public static final CommonVars DORIS_DATASOURCE_SYSTEM_QUERY_PARAM = + CommonVars.apply("linkis.ec.doris.datasource.systemQueryParam", ""); + + public static final CommonVars DORIS_RECONNECT_ENABLED = + CommonVars.apply("linkis.ec.doris.2pc.enabled", false, "two phase commit Whether to enable"); + + public static final CommonVars DORIS_STRIP_OUTER_ARRAY = + CommonVars.apply( + "linkis.ec.doris.strip.outer.array", + true, + "true indicates that the json data starts with an array object and flattens the array object, the default value is true, Refer to doris for strip_outer_array"); +} diff --git a/linkis-engineconn-plugins/doris/src/main/java/org/apache/linkis/engineplugin/doris/conf/DorisEngineConf.java b/linkis-engineconn-plugins/doris/src/main/java/org/apache/linkis/engineplugin/doris/conf/DorisEngineConf.java new file mode 100644 index 0000000000..04b0c386b3 --- /dev/null +++ b/linkis-engineconn-plugins/doris/src/main/java/org/apache/linkis/engineplugin/doris/conf/DorisEngineConf.java @@ -0,0 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.engineplugin.doris.conf; + +import org.apache.linkis.common.conf.Configuration; +import org.apache.linkis.governance.common.protocol.conf.RequestQueryEngineConfigWithGlobalConfig; +import org.apache.linkis.governance.common.protocol.conf.ResponseQueryConfig; +import org.apache.linkis.manager.label.entity.engine.EngineTypeLabel; +import org.apache.linkis.manager.label.entity.engine.UserCreatorLabel; +import org.apache.linkis.protocol.CacheableProtocol; +import org.apache.linkis.rpc.RPCMapCache; + +import java.util.Map; + +import scala.Tuple2; + +public class DorisEngineConf + extends RPCMapCache, String, String> { + + public DorisEngineConf() { + super(Configuration.CLOUD_CONSOLE_CONFIGURATION_SPRING_APPLICATION_NAME().getValue()); + } + + @Override + public CacheableProtocol createRequest(Tuple2 labelTuple) { + return new RequestQueryEngineConfigWithGlobalConfig(labelTuple._1(), labelTuple._2(), null); + } + + @Override + public Map createMap(Object obj) { + if (obj instanceof ResponseQueryConfig) { + ResponseQueryConfig response = (ResponseQueryConfig) obj; + return response.getKeyAndValue(); + } else { + return null; + } + } +} diff --git a/linkis-engineconn-plugins/doris/src/main/java/org/apache/linkis/engineplugin/doris/constant/DorisConstant.java b/linkis-engineconn-plugins/doris/src/main/java/org/apache/linkis/engineplugin/doris/constant/DorisConstant.java new file mode 100644 index 0000000000..6eb556c888 --- /dev/null +++ b/linkis-engineconn-plugins/doris/src/main/java/org/apache/linkis/engineplugin/doris/constant/DorisConstant.java @@ -0,0 +1,73 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.engineplugin.doris.constant; + +public class DorisConstant { + + public static final String STATUS = "Status"; + + public static final String TXN_ID = "TxnId"; + + public static final String TXN_ID_LOWER = "txn_id"; + + public static final String SUCCESS = "Success"; + + public static final String COMMIT = "commit"; + + public static final String ABORT = "abort"; + + public static final String COLUMNS = "columns"; + + public static final String LABEL = "label"; + + public static final String FORMAT = "format"; + + public static final String CSV = "csv"; + + public static final String JSON = "json"; + + public static final String PARQUET = "parquet"; + + public static final String ORC = "orc"; + + public static final String TWO_PHASE_COMMIT = "two_phase_commit"; + + public static final String STRIP_OUTER_ARRAY = "strip_outer_array"; + + public static final String COLUMN_SEPARATOR = "column_separator"; + + public static final String LINE_DELIMITER = "line_delimiter"; + + public static final String TXN_OPERATION = "txn_operation"; + + public static final Integer HTTP_SUCCEED = 200; + + public static final String DS_JDBC_HOST = "host"; + + public static final String DS_JDBC_PORT = "port"; + + public static final String DS_JDBC_DB_NAME = "databaseName"; + + public static final String DS_JDBC_DB_INSTANCE = "instance"; + + public static final String DS_JDBC_USERNAME = "username"; + + public static final String DS_JDBC_PASSWORD = "password"; + + public static final String DS_JDBC_PASSWORD_HIDE_VALUE = "******"; +} diff --git a/linkis-engineconn-plugins/doris/src/main/java/org/apache/linkis/engineplugin/doris/errorcode/DorisErrorCodeSummary.java b/linkis-engineconn-plugins/doris/src/main/java/org/apache/linkis/engineplugin/doris/errorcode/DorisErrorCodeSummary.java new file mode 100644 index 0000000000..6bd991c77b --- /dev/null +++ b/linkis-engineconn-plugins/doris/src/main/java/org/apache/linkis/engineplugin/doris/errorcode/DorisErrorCodeSummary.java @@ -0,0 +1,59 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.engineplugin.doris.errorcode; + +import org.apache.linkis.common.errorcode.ErrorCodeUtils; +import org.apache.linkis.common.errorcode.LinkisErrorCode; + +public enum DorisErrorCodeSummary implements LinkisErrorCode { + CHECK_DORIS_PARAMETER_FAILED(28501, "Failed to check the doris parameter(doris参数检查失败)"), + DORIS_TEST_CONNECTION_FAILED(28502, "The doris test connection failed(doris测试连接失败)"), + DORIS_STREAM_LOAD_FILE_PATH_NOT_FILE( + 28503, "The doris stream load file path must be a file(doris stream load file path必须是一个文件)"), + DORIS_CODE_IS_NOT_BLANK(28504, "Doris engine code cannot be empty(Doris引擎代码不能为空)"), + + DORIS_CODE_FAILED_TO_CONVERT_JSON( + 28505, "Doris code Failed to convert json(Doris code 转换json失败)"), + + DORIS_REQUIRED_PARAMETER_IS_NOT_BLANK( + 28506, "Doris required Parameter cannot be empty(Doris必填参数不能为空)"), + + DORIS_STREAM_LOAD_FILE_PATH_NOT_SUPPORTED_TYPE_FILE( + 28507, + "The doris stream load file path This file type is not currently supported(doris stream load file path目前不支持该文件类型)"); + + private final int errorCode; + + private final String errorDesc; + + DorisErrorCodeSummary(int errorCode, String errorDesc) { + ErrorCodeUtils.validateErrorCode(errorCode, 26000, 29999); + this.errorCode = errorCode; + this.errorDesc = errorDesc; + } + + @Override + public int getErrorCode() { + return errorCode; + } + + @Override + public String getErrorDesc() { + return errorDesc; + } +} diff --git a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/scheduler/cache/CacheOutputExecuteResponse.scala b/linkis-engineconn-plugins/doris/src/main/java/org/apache/linkis/engineplugin/doris/exception/DorisException.java similarity index 75% rename from linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/scheduler/cache/CacheOutputExecuteResponse.scala rename to linkis-engineconn-plugins/doris/src/main/java/org/apache/linkis/engineplugin/doris/exception/DorisException.java index 47a6ce9e9e..fa4d9b0989 100644 --- a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/scheduler/cache/CacheOutputExecuteResponse.scala +++ b/linkis-engineconn-plugins/doris/src/main/java/org/apache/linkis/engineplugin/doris/exception/DorisException.java @@ -15,10 +15,13 @@ * limitations under the License. */ -package org.apache.linkis.entrance.scheduler.cache +package org.apache.linkis.engineplugin.doris.exception; -import org.apache.linkis.scheduler.executer.OutputExecuteResponse +import org.apache.linkis.common.exception.ErrorException; -case class CacheOutputExecuteResponse(alias: String, output: String) extends OutputExecuteResponse { - override def getOutput: String = output +public class DorisException extends ErrorException { + + public DorisException(int errorCode, String message) { + super(errorCode, message); + } } diff --git a/linkis-engineconn-plugins/doris/src/main/java/org/apache/linkis/engineplugin/doris/exception/DorisParameterException.java b/linkis-engineconn-plugins/doris/src/main/java/org/apache/linkis/engineplugin/doris/exception/DorisParameterException.java new file mode 100644 index 0000000000..3a91e4d172 --- /dev/null +++ b/linkis-engineconn-plugins/doris/src/main/java/org/apache/linkis/engineplugin/doris/exception/DorisParameterException.java @@ -0,0 +1,27 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.engineplugin.doris.exception; + +import org.apache.linkis.common.exception.ErrorException; + +public class DorisParameterException extends ErrorException { + + public DorisParameterException(int errorCode, String message) { + super(errorCode, message); + } +} diff --git a/linkis-engineconn-plugins/doris/src/main/java/org/apache/linkis/engineplugin/doris/exception/DorisStreamLoadFileException.java b/linkis-engineconn-plugins/doris/src/main/java/org/apache/linkis/engineplugin/doris/exception/DorisStreamLoadFileException.java new file mode 100644 index 0000000000..a5b04061bf --- /dev/null +++ b/linkis-engineconn-plugins/doris/src/main/java/org/apache/linkis/engineplugin/doris/exception/DorisStreamLoadFileException.java @@ -0,0 +1,27 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.engineplugin.doris.exception; + +import org.apache.linkis.common.exception.ErrorException; + +public class DorisStreamLoadFileException extends ErrorException { + + public DorisStreamLoadFileException(int errorCode, String message) { + super(errorCode, message); + } +} diff --git a/linkis-engineconn-plugins/doris/src/main/java/org/apache/linkis/engineplugin/doris/executor/DorisDatasourceParser.java b/linkis-engineconn-plugins/doris/src/main/java/org/apache/linkis/engineplugin/doris/executor/DorisDatasourceParser.java new file mode 100644 index 0000000000..b9a0986d21 --- /dev/null +++ b/linkis-engineconn-plugins/doris/src/main/java/org/apache/linkis/engineplugin/doris/executor/DorisDatasourceParser.java @@ -0,0 +1,117 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.engineplugin.doris.executor; + +import org.apache.linkis.common.utils.JsonUtils; +import org.apache.linkis.datasource.client.impl.LinkisDataSourceRemoteClient; +import org.apache.linkis.datasource.client.request.GetInfoPublishedByDataSourceNameAction; +import org.apache.linkis.datasourcemanager.common.domain.DataSource; +import org.apache.linkis.engineplugin.doris.constant.DorisConstant; + +import org.apache.commons.collections.MapUtils; + +import java.util.HashMap; +import java.util.Map; + +import com.fasterxml.jackson.core.JsonProcessingException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class DorisDatasourceParser { + private static final Logger logger = LoggerFactory.getLogger(DorisDatasourceParser.class); + + public static Map queryDatasourceInfoByName( + String datasourceName, String username, String system) { + logger.info( + "Starting query [" + + system + + ", " + + username + + ", " + + datasourceName + + "] datasource info ......"); + LinkisDataSourceRemoteClient dataSourceClient = new LinkisDataSourceRemoteClient(); + DataSource dataSource = + dataSourceClient + .getInfoPublishedByDataSourceName( + GetInfoPublishedByDataSourceNameAction.builder() + .setSystem(system) + .setDataSourceName(datasourceName) + .setUser(username) + .build()) + .getDataSource(); + + return queryDatasourceParam(datasourceName, dataSource); + } + + private static Map queryDatasourceParam( + String datasourceName, DataSource dataSource) { + Map paramMap = new HashMap<>(); + + if (dataSource == null) { + logger.warn("Doris dataSource is null: {}", datasourceName); + return paramMap; + } + + if (dataSource.isExpire()) { + logger.warn("Doris dataSource of datasource name: {} is expired", datasourceName); + return paramMap; + } + + Map connectParams = dataSource.getConnectParams(); + if (MapUtils.isEmpty(connectParams)) { + logger.warn("Doris dataSource connectParams is empty: {}", datasourceName); + return paramMap; + } + + paramMap.put( + DorisConstant.DS_JDBC_HOST, + String.valueOf(connectParams.getOrDefault(DorisConstant.DS_JDBC_HOST, ""))); + paramMap.put( + DorisConstant.DS_JDBC_PORT, + String.valueOf(connectParams.getOrDefault(DorisConstant.DS_JDBC_PORT, ""))); + paramMap.put( + DorisConstant.DS_JDBC_USERNAME, + String.valueOf(connectParams.getOrDefault(DorisConstant.DS_JDBC_USERNAME, ""))); + paramMap.put( + DorisConstant.DS_JDBC_PASSWORD, + String.valueOf(connectParams.getOrDefault(DorisConstant.DS_JDBC_PASSWORD, ""))); + paramMap.put( + DorisConstant.DS_JDBC_DB_NAME, + String.valueOf( + connectParams.getOrDefault( + DorisConstant.DS_JDBC_DB_NAME, + connectParams.getOrDefault(DorisConstant.DS_JDBC_DB_INSTANCE, "")))); + + try { + HashMap printMap = new HashMap<>(); + printMap.putAll(paramMap); + + // To hide the password and prevent leaks + if (printMap.containsKey(DorisConstant.DS_JDBC_PASSWORD)) { + printMap.put(DorisConstant.DS_JDBC_PASSWORD, DorisConstant.DS_JDBC_PASSWORD_HIDE_VALUE); + } + String printMapString = JsonUtils.jackson().writeValueAsString(printMap); + logger.info("Load dataSource param: {}", printMapString); + } catch (JsonProcessingException e) { + + } + + return paramMap; + } +} diff --git a/linkis-engineconn-plugins/doris/src/main/java/org/apache/linkis/engineplugin/doris/executor/DorisEngineConnExecutor.java b/linkis-engineconn-plugins/doris/src/main/java/org/apache/linkis/engineplugin/doris/executor/DorisEngineConnExecutor.java new file mode 100644 index 0000000000..dd0a88dddf --- /dev/null +++ b/linkis-engineconn-plugins/doris/src/main/java/org/apache/linkis/engineplugin/doris/executor/DorisEngineConnExecutor.java @@ -0,0 +1,727 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.engineplugin.doris.executor; + +import org.apache.linkis.common.io.MetaData; +import org.apache.linkis.common.io.Record; +import org.apache.linkis.common.io.resultset.ResultSetWriter; +import org.apache.linkis.common.log.LogUtils; +import org.apache.linkis.common.utils.JsonUtils; +import org.apache.linkis.common.utils.OverloadUtils; +import org.apache.linkis.engineconn.computation.executor.entity.EngineConnTask; +import org.apache.linkis.engineconn.computation.executor.execute.ConcurrentComputationExecutor; +import org.apache.linkis.engineconn.computation.executor.execute.EngineExecutionContext; +import org.apache.linkis.engineconn.core.EngineConnObject; +import org.apache.linkis.engineplugin.doris.conf.DorisConfiguration; +import org.apache.linkis.engineplugin.doris.conf.DorisEngineConf; +import org.apache.linkis.engineplugin.doris.constant.DorisConstant; +import org.apache.linkis.engineplugin.doris.errorcode.DorisErrorCodeSummary; +import org.apache.linkis.engineplugin.doris.exception.DorisException; +import org.apache.linkis.engineplugin.doris.exception.DorisParameterException; +import org.apache.linkis.engineplugin.doris.exception.DorisStreamLoadFileException; +import org.apache.linkis.engineplugin.doris.util.DorisUtils; +import org.apache.linkis.manager.common.entity.resource.CommonNodeResource; +import org.apache.linkis.manager.common.entity.resource.LoadResource; +import org.apache.linkis.manager.common.entity.resource.NodeResource; +import org.apache.linkis.manager.engineplugin.common.util.NodeResourceUtils; +import org.apache.linkis.manager.label.entity.Label; +import org.apache.linkis.manager.label.entity.engine.EngineTypeLabel; +import org.apache.linkis.manager.label.entity.engine.UserCreatorLabel; +import org.apache.linkis.protocol.engine.JobProgressInfo; +import org.apache.linkis.rpc.Sender; +import org.apache.linkis.scheduler.executer.ErrorExecuteResponse; +import org.apache.linkis.scheduler.executer.ExecuteResponse; +import org.apache.linkis.scheduler.executer.SuccessExecuteResponse; +import org.apache.linkis.storage.LineMetaData; +import org.apache.linkis.storage.LineRecord; +import org.apache.linkis.storage.resultset.ResultSetFactory; + +import org.apache.commons.codec.binary.Base64; +import org.apache.commons.collections4.MapUtils; +import org.apache.commons.io.FilenameUtils; +import org.apache.commons.io.IOUtils; +import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.exception.ExceptionUtils; +import org.apache.http.HttpHeaders; +import org.apache.http.client.methods.CloseableHttpResponse; +import org.apache.http.client.methods.HttpPut; +import org.apache.http.entity.FileEntity; +import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.impl.client.DefaultRedirectStrategy; +import org.apache.http.impl.client.HttpClients; +import org.apache.http.util.EntityUtils; + +import org.springframework.util.CollectionUtils; + +import java.io.File; +import java.io.IOException; +import java.net.HttpURLConnection; +import java.net.URL; +import java.nio.charset.StandardCharsets; +import java.util.*; +import java.util.concurrent.ConcurrentHashMap; +import java.util.stream.Collectors; + +import scala.Tuple2; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.core.type.TypeReference; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import static org.apache.linkis.engineplugin.doris.constant.DorisConstant.*; + +/** + * Inspired by: + * https://github.com/apache/doris/blob/master/samples/stream_load/java/DorisStreamLoad.java + */ +public class DorisEngineConnExecutor extends ConcurrentComputationExecutor { + + private static final Logger logger = LoggerFactory.getLogger(DorisEngineConnExecutor.class); + private int id; + + private List> executorLabels = new ArrayList<>(2); + private Map threadCache = new ConcurrentHashMap<>(); + + private Map configMap = new HashMap<>(); + + private static final String DORIS_LABEL_PREFIX = "linkis_doris_"; + + public static final String DORIS_URL_BOOTSTRAP = "http://%s:%s/api/bootstrap"; + + public static final String DORIS_URL_STREAM_LOAD = "http://%s:%s/api/%s/%s/_stream_load"; + + public static final String DORIS_URL_STREAM_LOAD_2PC = "http://%s:%s/api/%s/%s/_stream_load_2pc"; + + private String dorisHost; + private String dorisDatabase; + + private String datasourceDatabase; + private String dorisTable; + private String dorisUsername; + private String dorisPassword; + + private String dorisStreamLoadFilePath; + private Integer dorisHttpPort; + + private Integer dorisJdbcPort; + + private CloseableHttpClient client; + + public DorisEngineConnExecutor(int outputPrintLimit, int id) { + super(outputPrintLimit); + this.id = id; + } + + @Override + public void init() { + super.init(); + } + + @Override + public ExecuteResponse execute(EngineConnTask engineConnTask) { + Optional> userCreatorLabelOp = + Arrays.stream(engineConnTask.getLables()) + .filter(label -> label instanceof UserCreatorLabel) + .findFirst(); + Optional> engineTypeLabelOp = + Arrays.stream(engineConnTask.getLables()) + .filter(label -> label instanceof EngineTypeLabel) + .findFirst(); + + if (userCreatorLabelOp.isPresent() && engineTypeLabelOp.isPresent()) { + UserCreatorLabel userCreatorLabel = (UserCreatorLabel) userCreatorLabelOp.get(); + EngineTypeLabel engineTypeLabel = (EngineTypeLabel) engineTypeLabelOp.get(); + + Map cacheMap = + new DorisEngineConf().getCacheMap(new Tuple2<>(userCreatorLabel, engineTypeLabel)); + if (MapUtils.isNotEmpty(cacheMap)) { + configMap.putAll(cacheMap); + } + } + + Map taskParams = engineConnTask.getProperties(); + + if (MapUtils.isNotEmpty(taskParams)) { + taskParams.entrySet().stream() + .filter(entry -> entry.getValue() != null) + .forEach(entry -> configMap.put(entry.getKey(), String.valueOf(entry.getValue()))); + } + + this.client = + HttpClients.custom() + .setRedirectStrategy( + new DefaultRedirectStrategy() { + @Override + protected boolean isRedirectable(String method) { + return true; + } + }) + .build(); + return super.execute(engineConnTask); + } + + @Override + public ExecuteResponse executeLine(EngineExecutionContext engineExecutorContext, String code) { + checkParameter(engineExecutorContext); + + String realCode; + if (StringUtils.isBlank(code)) { + throw new DorisException( + DorisErrorCodeSummary.DORIS_CODE_IS_NOT_BLANK.getErrorCode(), + DorisErrorCodeSummary.DORIS_CODE_IS_NOT_BLANK.getErrorDesc()); + } else { + realCode = code.trim(); + } + logger.info("Doris engine begins to run code:\n {}", realCode); + + checkRequiredParameter(realCode); + + String testConnectionUrl = String.format(DORIS_URL_BOOTSTRAP, dorisHost, dorisHttpPort); + + if (!testConnection(testConnectionUrl)) { + logger.error("Test connection failed: {}", testConnectionUrl); + throw new DorisException( + DorisErrorCodeSummary.DORIS_TEST_CONNECTION_FAILED.getErrorCode(), + DorisErrorCodeSummary.DORIS_TEST_CONNECTION_FAILED.getErrorDesc()); + } + + String taskId = engineExecutorContext.getJobId().get(); + + initialStatusUpdates(taskId, engineExecutorContext); + + threadCache.put(taskId, Thread.currentThread()); + + CloseableHttpResponse response = null; + Boolean executeResponse = false; + try { + response = streamLoad(engineExecutorContext); + } catch (Exception e) { + String errorMessage = ExceptionUtils.getStackTrace(e); + logger.error("Doris engine execute failed : {}", errorMessage); + engineExecutorContext.appendStdout(LogUtils.generateERROR(errorMessage)); + return new ErrorExecuteResponse(errorMessage, null); + } + + String resultMessage = entitytoString(response); + StringBuilder resultMessageStringBuilder = new StringBuilder(); + resultMessageStringBuilder.append(resultMessage); + + logger.info("Doris stream load execution result: {}", resultMessage); + Map resultMap = responseToMap(resultMessage); + int statusCode = response.getStatusLine().getStatusCode(); + Boolean enabled2PC = DorisConfiguration.DORIS_RECONNECT_ENABLED.getValue(configMap); + + // If two phase commit is enabled, commit if executed successfully, abort otherwise + if (statusCode == HTTP_SUCCEED && isSuccess(resultMap)) { + executeResponse = true; + if (enabled2PC && resultMap.containsKey(TXN_ID)) { + String commitMessage = "doris begin commit"; + logger.info(commitMessage); + engineExecutorContext.appendStdout(commitMessage); + executeResponse = + dorisCommitOrAbort(resultMap.get(TXN_ID), COMMIT, resultMessageStringBuilder); + } + } else { + executeResponse = false; + if (enabled2PC && resultMap.containsKey(TXN_ID)) { + String abortMessage = "doris stream load failed, begin abort"; + logger.error(abortMessage); + engineExecutorContext.appendStdout(abortMessage); + dorisCommitOrAbort(resultMap.get(TXN_ID), ABORT, resultMessageStringBuilder); + } + } + + engineExecutorContext.appendStdout(resultMessageStringBuilder.toString()); + ResultSetWriter resultSetWriter = + engineExecutorContext.createResultSetWriter(ResultSetFactory.TEXT_TYPE); + try { + resultSetWriter.addMetaData(new LineMetaData()); + resultSetWriter.addRecord(new LineRecord(resultMessageStringBuilder.toString())); + } catch (IOException e) { + logger.error("Failed to get the task result"); + } finally { + IOUtils.closeQuietly(resultSetWriter); + } + + if (executeResponse) { + return new SuccessExecuteResponse(); + } else { + return new ErrorExecuteResponse(resultMessageStringBuilder.toString(), null); + } + } + + private CloseableHttpResponse streamLoad(EngineExecutionContext engineExecutorContext) + throws Exception { + String loadUrl = + String.format(DORIS_URL_STREAM_LOAD, dorisHost, dorisHttpPort, dorisDatabase, dorisTable); + + logger.info("Doris engine stream load begins to run loadUrl:\n {}", loadUrl); + engineExecutorContext.appendStdout( + String.format("Doris engine stream load begins to run loadUrl:\n %s", loadUrl)); + + HttpPut httpPut = new HttpPut(loadUrl); + + // Set the doris configuration, which has a low priority and will be overwritten by other + // configurations + String dorisConf = DorisConfiguration.DORIS_CONF.getValue(configMap); + if (StringUtils.isNotBlank(dorisConf)) { + String[] confs = dorisConf.split(","); + for (String conf : confs) { + String[] keyValue = conf.split(":"); + if (keyValue.length == 2) { + String key = keyValue[0]; + String value = keyValue[1]; + httpPut.setHeader(key, value); + logger.info("doris set param {} : {}", key, value); + } + } + } + + addCommonHeader(httpPut); + + String dorisColumns = DorisConfiguration.DORIS_COLUMNS.getValue(configMap); + + if (StringUtils.isBlank(dorisColumns)) { + List dorisCloumns = + DorisUtils.getDorisCloumns( + dorisHost, dorisJdbcPort, dorisUsername, dorisPassword, dorisDatabase, dorisTable); + if (org.apache.commons.collections.CollectionUtils.isNotEmpty(dorisCloumns)) { + dorisColumns = + String.join( + ",", + dorisCloumns.stream() + .map(f -> String.format("`%s`", f)) + .collect(Collectors.toList())); + } + } + + // dorisColumns may fail to obtain metadata and is empty + if (StringUtils.isNotBlank(dorisColumns)) { + httpPut.setHeader(COLUMNS, dorisColumns); + logger.info("doris set param {} : {}", COLUMNS, dorisColumns); + } + + // the label header is optional, not necessary + // use label header can ensure at most once semantics + String dorisLabel = DorisConfiguration.DORIS_LABEL.getValue(configMap); + if (StringUtils.isBlank(dorisLabel)) { + dorisLabel = DORIS_LABEL_PREFIX + UUID.randomUUID(); + } + httpPut.setHeader(LABEL, dorisLabel); + logger.info("doris set param {} : {}", LABEL, dorisLabel); + + File dorisStreamLoadFile = new File(dorisStreamLoadFilePath); + if (!dorisStreamLoadFile.isFile()) { + throw new DorisStreamLoadFileException( + DorisErrorCodeSummary.DORIS_STREAM_LOAD_FILE_PATH_NOT_FILE.getErrorCode(), + DorisErrorCodeSummary.DORIS_STREAM_LOAD_FILE_PATH_NOT_FILE.getErrorDesc()); + } + + String fileExtension = FilenameUtils.getExtension(dorisStreamLoadFilePath); + + // Currently only csv、json、parquet、orc format are supported + if (!isSupportedType(fileExtension)) { + logger.error( + "The supported types are csv, json, parquet, and orc,This file type is not currently supported: {}", + fileExtension); + throw new DorisStreamLoadFileException( + DorisErrorCodeSummary.DORIS_STREAM_LOAD_FILE_PATH_NOT_SUPPORTED_TYPE_FILE.getErrorCode(), + DorisErrorCodeSummary.DORIS_STREAM_LOAD_FILE_PATH_NOT_SUPPORTED_TYPE_FILE.getErrorDesc()); + } + + httpPut.setHeader(FORMAT, fileExtension); + + Boolean enabled2PC = DorisConfiguration.DORIS_RECONNECT_ENABLED.getValue(configMap); + httpPut.setHeader(TWO_PHASE_COMMIT, String.valueOf(enabled2PC)); + logger.info("doris set param {} : {}", TWO_PHASE_COMMIT, enabled2PC); + + if (fileExtension.equals(JSON)) { + Boolean stripOuterArray = DorisConfiguration.DORIS_STRIP_OUTER_ARRAY.getValue(configMap); + httpPut.setHeader(STRIP_OUTER_ARRAY, String.valueOf(stripOuterArray)); + logger.info("doris set param {} : {}", STRIP_OUTER_ARRAY, stripOuterArray); + } + + String dorisColumnSeparator = DorisConfiguration.DORIS_COLUMN_SEPARATOR.getValue(configMap); + httpPut.setHeader(COLUMN_SEPARATOR, dorisColumnSeparator); + logger.info("doris set param {} : {}", COLUMN_SEPARATOR, dorisColumnSeparator); + + String dorisLineDelimiter = DorisConfiguration.DORIS_LINE_DELIMITER.getValue(configMap); + httpPut.setHeader(LINE_DELIMITER, dorisLineDelimiter); + logger.info("doris set param {} : {}", LINE_DELIMITER, dorisLineDelimiter); + + FileEntity entity = new FileEntity(dorisStreamLoadFile); + httpPut.setEntity(entity); + engineExecutorContext.appendStdout( + String.format("doris stread load file path: %s", dorisStreamLoadFile.getAbsolutePath())); + + String allHeaders = Arrays.toString(httpPut.getAllHeaders()); + logger.info("doris param: {}", allHeaders); + engineExecutorContext.appendStdout(String.format("doris param: %s", allHeaders)); + + return client.execute(httpPut); + } + + private boolean isSupportedType(String fileExtension) { + if (StringUtils.isBlank(fileExtension)) { + return false; + } + if (fileExtension.equals(CSV) + || fileExtension.equals(JSON) + || fileExtension.equals(PARQUET) + || fileExtension.equals(ORC)) { + return true; + } + return false; + } + + private void checkParameter(EngineExecutionContext engineExecutorContext) { + this.dorisHost = DorisConfiguration.DORIS_HOST.getValue(configMap); + this.dorisUsername = DorisConfiguration.DORIS_USER_NAME.getValue(configMap); + this.dorisHttpPort = DorisConfiguration.DORIS_HTTP_PORT.getValue(configMap); + this.dorisPassword = DorisConfiguration.DORIS_PASSWORD.getValue(configMap); + this.dorisJdbcPort = DorisConfiguration.DORIS_JDBC_PORT.getValue(configMap); + + String dorisDatasourceName = DorisConfiguration.DORIS_DATASOURCE.getValue(configMap); + + // Data source parameters fail to be obtained, and task running cannot be affected + // The datasource param overwrites the DorisConfiguration param + try { + if (StringUtils.isNotBlank(dorisDatasourceName)) { + String dorisSystemQueryParam = + DorisConfiguration.DORIS_DATASOURCE_SYSTEM_QUERY_PARAM.getValue(configMap); + String execSqlUser = getExecSqlUser(engineExecutorContext); + + Map dataSourceParamMap = + DorisDatasourceParser.queryDatasourceInfoByName( + dorisDatasourceName, execSqlUser, dorisSystemQueryParam); + + if (MapUtils.isNotEmpty(dataSourceParamMap)) { + if (dataSourceParamMap.containsKey(DS_JDBC_HOST) + && StringUtils.isNotBlank(dataSourceParamMap.get(DS_JDBC_HOST))) { + this.dorisHost = dataSourceParamMap.get(DS_JDBC_HOST); + } + + if (dataSourceParamMap.containsKey(DS_JDBC_USERNAME) + && StringUtils.isNotBlank(dataSourceParamMap.get(DS_JDBC_USERNAME))) { + this.dorisUsername = dataSourceParamMap.get(DS_JDBC_USERNAME); + } + + if (dataSourceParamMap.containsKey(DS_JDBC_PASSWORD)) { + this.dorisPassword = dataSourceParamMap.get(DS_JDBC_PASSWORD); + } + + if (dataSourceParamMap.containsKey(DS_JDBC_PORT) + && StringUtils.isNotBlank(dataSourceParamMap.get(DS_JDBC_PORT))) { + this.dorisJdbcPort = Integer.valueOf(dataSourceParamMap.get(DS_JDBC_PORT)); + } + + if (dataSourceParamMap.containsKey(DS_JDBC_DB_NAME) + && StringUtils.isNotBlank(dataSourceParamMap.get(DS_JDBC_DB_NAME))) { + this.datasourceDatabase = dataSourceParamMap.get(DS_JDBC_DB_NAME); + } + } else { + logger.warn( + "Doris dataSource {} param is null, Skip get doris dataSource parameters", + dorisDatasourceName); + } + } + } catch (Exception e) { + logger.error("get doris dataSource {} param failed", dorisDatasourceName, e); + } + + if (StringUtils.isBlank(this.dorisHost) + || StringUtils.isBlank(this.dorisUsername) + || this.dorisHttpPort == null) { + logger.error("Doris check param failed."); + throw new DorisParameterException( + DorisErrorCodeSummary.CHECK_DORIS_PARAMETER_FAILED.getErrorCode(), + DorisErrorCodeSummary.CHECK_DORIS_PARAMETER_FAILED.getErrorDesc()); + } + + logger.info( + "Doris parameter dorisHost: {}, dorisUsername: {}, dorisPassword: {}, dorisHttpPort: {}, dorisJdbcPort: {}.", + this.dorisHost, + this.dorisUsername, + DorisConstant.DS_JDBC_PASSWORD_HIDE_VALUE, + this.dorisHttpPort, + this.dorisJdbcPort); + } + + private String getExecSqlUser(EngineExecutionContext engineExecutionContext) { + UserCreatorLabel userCreatorLabel = + (UserCreatorLabel) + Arrays.stream(engineExecutionContext.getLabels()) + .filter(label -> label instanceof UserCreatorLabel) + .findFirst() + .orElse(null); + if (userCreatorLabel != null) { + return userCreatorLabel.getUser(); + } + return null; + } + + private void checkRequiredParameter(String code) { + Map codeMap = new HashMap<>(); + + try { + codeMap = + JsonUtils.jackson().readValue(code, new TypeReference>() {}); + } catch (JsonProcessingException e) { + throw new DorisException( + DorisErrorCodeSummary.DORIS_CODE_FAILED_TO_CONVERT_JSON.getErrorCode(), + DorisErrorCodeSummary.DORIS_CODE_FAILED_TO_CONVERT_JSON.getErrorDesc()); + } + + this.dorisStreamLoadFilePath = + codeMap.getOrDefault(DorisConfiguration.DORIS_STREAM_LOAD_FILE_PATH.key(), ""); + this.dorisTable = codeMap.getOrDefault(DorisConfiguration.DORIS_TABLE.key(), ""); + this.dorisDatabase = codeMap.getOrDefault(DorisConfiguration.DORIS_DATABASE.key(), ""); + + String dorisDatasourceName = DorisConfiguration.DORIS_DATASOURCE.getValue(configMap); + // The datasource param overwrites the DorisConfiguration param + if (StringUtils.isNotBlank(dorisDatasourceName) && StringUtils.isNotBlank(datasourceDatabase)) { + this.dorisDatabase = datasourceDatabase; + } + + if (StringUtils.isBlank(this.dorisStreamLoadFilePath) + || StringUtils.isBlank(this.dorisDatabase) + || StringUtils.isBlank(this.dorisTable)) { + logger.error( + "Check whether `{}`, `{}`, and `{}` are included in code json", + DorisConfiguration.DORIS_STREAM_LOAD_FILE_PATH.key(), + DorisConfiguration.DORIS_DATABASE.key(), + DorisConfiguration.DORIS_TABLE.key()); + throw new DorisException( + DorisErrorCodeSummary.DORIS_REQUIRED_PARAMETER_IS_NOT_BLANK.getErrorCode(), + DorisErrorCodeSummary.DORIS_REQUIRED_PARAMETER_IS_NOT_BLANK.getErrorDesc()); + } + + logger.info( + "Doris parameter dorisStreamLoadFilePath: {}, dorisDatabase: {}, dorisTable: {}.", + this.dorisStreamLoadFilePath, + this.dorisDatabase, + this.dorisTable); + } + + private boolean isSuccess(Map map) { + if (org.apache.commons.collections.MapUtils.isEmpty(map)) { + return false; + } + + if (map.containsKey(STATUS) && map.get(STATUS).equalsIgnoreCase(SUCCESS)) { + return true; + } + + // Sometimes Status is uppercase and sometimes lowercase + if (map.containsKey(STATUS.toLowerCase()) + && map.get(STATUS.toLowerCase()).equalsIgnoreCase(SUCCESS)) { + return true; + } + + return false; + } + + /** After the two phase commit is enabled, you can use it to commit or abort */ + private boolean dorisCommitOrAbort( + String id, String type, StringBuilder resultMessageStringBuilder) { + String load2PCUrl = + String.format( + DORIS_URL_STREAM_LOAD_2PC, dorisHost, dorisHttpPort, dorisDatabase, dorisTable); + HttpPut commmitHttpPut = new HttpPut(load2PCUrl); + addCommonHeader(commmitHttpPut); + commmitHttpPut.setHeader(TXN_ID_LOWER, id); + commmitHttpPut.setHeader(TXN_OPERATION, type); + + CloseableHttpResponse commmitResponse = null; + try { + commmitResponse = client.execute(commmitHttpPut); + } catch (IOException e) { + logger.error("doris {} failed", type, e); + return false; + } + String commmitLoadResult = entitytoString(commmitResponse); + logger.info("Doris stream load {} execution result: {}", type, commmitLoadResult); + resultMessageStringBuilder.append("\r\n").append(commmitLoadResult); + + Map commmitResultMap = responseToMap(commmitLoadResult); + int statusCode = commmitResponse.getStatusLine().getStatusCode(); + + if (statusCode == HTTP_SUCCEED && isSuccess(commmitResultMap)) { + return true; + } + return false; + } + + private static String entitytoString(CloseableHttpResponse response) { + String loadResult = ""; + if (response.getEntity() != null) { + try { + loadResult = EntityUtils.toString(response.getEntity()); + } catch (IOException e) { + logger.error("Doris httpResponse entity conversion to string failed", e); + } + } + return loadResult; + } + + private void addCommonHeader(HttpPut httpPut) { + if (httpPut == null) return; + httpPut.setHeader(HttpHeaders.EXPECT, "100-continue"); + httpPut.setHeader(HttpHeaders.AUTHORIZATION, basicAuthHeader(dorisUsername, dorisPassword)); + } + + private Map responseToMap(String response) { + Map resultMap = new HashMap<>(); + + if (StringUtils.isBlank(response)) { + return resultMap; + } + + try { + resultMap = + JsonUtils.jackson().readValue(response, new TypeReference>() {}); + } catch (JsonProcessingException e) { + logger.error("doris response to map failed", e); + return resultMap; + } + + return resultMap; + } + + private boolean testConnection(String testUrl) { + try { + URL url = new URL(testUrl); + HttpURLConnection connection = (HttpURLConnection) url.openConnection(); + connection.setConnectTimeout(5000); + connection.setRequestMethod("GET"); + connection.connect(); + int responseCode = connection.getResponseCode(); + if (responseCode == HTTP_SUCCEED) { + return true; + } + } catch (Exception e) { + return false; + } + return false; + } + + private String basicAuthHeader(String username, String password) { + String tobeEncode = username + ":" + password; + byte[] encoded = Base64.encodeBase64(tobeEncode.getBytes(StandardCharsets.UTF_8)); + return "Basic " + new String(encoded); + } + + @Override + public ExecuteResponse executeCompletely( + EngineExecutionContext engineExecutorContext, String code, String completedLine) { + return null; + } + + @Override + public float progress(String taskID) { + return 0.0f; + } + + @Override + public JobProgressInfo[] getProgressInfo(String taskID) { + return new JobProgressInfo[0]; + } + + @Override + public void killTask(String taskId) { + Thread thread = threadCache.remove(taskId); + if (null != thread) { + thread.interrupt(); + } + super.killTask(taskId); + } + + @Override + public List> getExecutorLabels() { + return executorLabels; + } + + @Override + public void setExecutorLabels(List> labels) { + if (!CollectionUtils.isEmpty(labels)) { + executorLabels.clear(); + executorLabels.addAll(labels); + } + } + + @Override + public boolean supportCallBackLogs() { + return false; + } + + @Override + public NodeResource requestExpectedResource(NodeResource expectedResource) { + return null; + } + + @Override + public NodeResource getCurrentNodeResource() { + NodeResourceUtils.appendMemoryUnitIfMissing( + EngineConnObject.getEngineCreationContext().getOptions()); + + CommonNodeResource resource = new CommonNodeResource(); + LoadResource usedResource = new LoadResource(OverloadUtils.getProcessMaxMemory(), 1); + resource.setUsedResource(usedResource); + return resource; + } + + @Override + public String getId() { + return Sender.getThisServiceInstance().getInstance() + "_" + id; + } + + @Override + public int getConcurrentLimit() { + return DorisConfiguration.ENGINE_CONCURRENT_LIMIT.getValue(); + } + + private void initialStatusUpdates(String taskId, EngineExecutionContext engineExecutorContext) { + engineExecutorContext.pushProgress(progress(taskId), getProgressInfo(taskId)); + } + + @Override + public void killAll() { + Iterator iterator = threadCache.values().iterator(); + while (iterator.hasNext()) { + Thread thread = iterator.next(); + if (thread != null) { + thread.interrupt(); + } + } + threadCache.clear(); + } + + @Override + public void close() { + killAll(); + try { + if (client != null) { + this.client.close(); + } + } catch (IOException e) { + logger.warn("close doris HttpClient failed"); + } + super.close(); + } +} diff --git a/linkis-engineconn-plugins/doris/src/main/java/org/apache/linkis/engineplugin/doris/util/DorisUtils.java b/linkis-engineconn-plugins/doris/src/main/java/org/apache/linkis/engineplugin/doris/util/DorisUtils.java new file mode 100644 index 0000000000..2d6d560d77 --- /dev/null +++ b/linkis-engineconn-plugins/doris/src/main/java/org/apache/linkis/engineplugin/doris/util/DorisUtils.java @@ -0,0 +1,107 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.engineplugin.doris.util; + +import java.sql.*; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class DorisUtils { + + private static final Logger logger = LoggerFactory.getLogger(DorisUtils.class); + + private static final String JDBC_URL = "jdbc:mysql://%s:%s/%s"; + + public static List getDorisCloumns( + String host, Integer port, String username, String password, String database, String table) { + String url = String.format(JDBC_URL, host, port, database); + + Connection connecion = getConnecion(username, password, url); + if (connecion == null) { + return Collections.emptyList(); + } + + String columnSql = "SELECT * FROM `" + database + "`.`" + table + "` WHERE 1 = 2"; + PreparedStatement ps = null; + ResultSet rs = null; + ResultSetMetaData meta = null; + List columns = new ArrayList<>(); + + try { + ps = connecion.prepareStatement(columnSql); + rs = ps.executeQuery(); + meta = rs.getMetaData(); + int columnCount = meta.getColumnCount(); + for (int i = 1; i < columnCount + 1; i++) { + columns.add(meta.getColumnName(i)); + } + } catch (SQLException e) { + logger.error("getDorisCloumns failed", e); + return columns; + } finally { + closeResource(connecion, ps, rs); + } + return columns; + } + + public static void closeResource( + Connection connection, Statement statement, ResultSet resultSet) { + try { + if (null != resultSet && !resultSet.isClosed()) { + resultSet.close(); + } + if (null != statement && !statement.isClosed()) { + statement.close(); + } + if (null != connection && !connection.isClosed()) { + connection.close(); + } + } catch (SQLException e) { + logger.warn("Fail to release resource [" + e.getMessage() + "]", e); + } + } + + private static Connection getConnecion(String username, String password, String url) { + Connection connection = null; + + try { + Class.forName("com.mysql.jdbc.Driver"); + } catch (ClassNotFoundException e) { + try { + Class.forName("com.mysql.cj.jdbc.Driver"); + } catch (ClassNotFoundException ex) { + logger.warn( + "The mysql driver does not exist, mysql driver is used to fetch the table column. If you need to use this feature, you need to place the mysql jar into the doris ec lib. ClassNotFoundException: {}", + ex.getMessage()); + return connection; + } + } + + try { + connection = DriverManager.getConnection(url, username, password); + } catch (Exception e) { + logger.warn( + "getConnecion failed,please check whether the connection parameters are correct.", e); + } + return connection; + } +} diff --git a/linkis-engineconn-plugins/doris/src/main/resources/linkis-engineconn.properties b/linkis-engineconn-plugins/doris/src/main/resources/linkis-engineconn.properties new file mode 100644 index 0000000000..fd897c71fd --- /dev/null +++ b/linkis-engineconn-plugins/doris/src/main/resources/linkis-engineconn.properties @@ -0,0 +1,23 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +wds.linkis.server.version=v1 +#wds.linkis.engineconn.debug.enable=true +#wds.linkis.keytab.enable=true +wds.linkis.engineconn.plugin.default.class=org.apache.linkis.engineplugin.doris.DorisEngineConnPlugin + +wds.linkis.engineconn.support.parallelism=true + +wds.linkis.engineconn.max.free.time=0 \ No newline at end of file diff --git a/linkis-engineconn-plugins/doris/src/main/resources/log4j2.xml b/linkis-engineconn-plugins/doris/src/main/resources/log4j2.xml new file mode 100644 index 0000000000..b6f2fc895c --- /dev/null +++ b/linkis-engineconn-plugins/doris/src/main/resources/log4j2.xml @@ -0,0 +1,95 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/linkis-engineconn-plugins/doris/src/main/scala/org/apache/linkis/engineplugin/doris/factory/DorisEngineConnFactory.scala b/linkis-engineconn-plugins/doris/src/main/scala/org/apache/linkis/engineplugin/doris/factory/DorisEngineConnFactory.scala new file mode 100644 index 0000000000..cbc75dfd7e --- /dev/null +++ b/linkis-engineconn-plugins/doris/src/main/scala/org/apache/linkis/engineplugin/doris/factory/DorisEngineConnFactory.scala @@ -0,0 +1,44 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.engineplugin.doris.factory + +import org.apache.linkis.engineconn.common.creation.EngineCreationContext +import org.apache.linkis.engineconn.common.engineconn.EngineConn +import org.apache.linkis.engineconn.computation.executor.creation.ComputationSingleExecutorEngineConnFactory +import org.apache.linkis.engineconn.executor.entity.LabelExecutor +import org.apache.linkis.engineplugin.doris.conf.DorisConfiguration +import org.apache.linkis.engineplugin.doris.executor.DorisEngineConnExecutor +import org.apache.linkis.manager.label.entity.engine.{EngineType, RunType} +import org.apache.linkis.manager.label.entity.engine.EngineType.EngineType +import org.apache.linkis.manager.label.entity.engine.RunType.RunType + +class DorisEngineConnFactory extends ComputationSingleExecutorEngineConnFactory { + + override def newExecutor( + id: Int, + engineCreationContext: EngineCreationContext, + engineConn: EngineConn + ): LabelExecutor = { + new DorisEngineConnExecutor(DorisConfiguration.ENGINE_DEFAULT_LIMIT.getValue, id) + } + + override protected def getEngineConnType: EngineType = EngineType.DORIS + + override protected def getRunType: RunType = RunType.DORIS + +} diff --git a/linkis-engineconn-plugins/flink/flink-core/src/main/java/org/apache/linkis/engineconnplugin/flink/client/deployment/ClusterDescriptorAdapter.java b/linkis-engineconn-plugins/flink/flink-core/src/main/java/org/apache/linkis/engineconnplugin/flink/client/deployment/ClusterDescriptorAdapter.java index 6930a43f05..594f8dd98d 100644 --- a/linkis-engineconn-plugins/flink/flink-core/src/main/java/org/apache/linkis/engineconnplugin/flink/client/deployment/ClusterDescriptorAdapter.java +++ b/linkis-engineconn-plugins/flink/flink-core/src/main/java/org/apache/linkis/engineconnplugin/flink/client/deployment/ClusterDescriptorAdapter.java @@ -51,7 +51,7 @@ public abstract class ClusterDescriptorAdapter implements Closeable { public static final long CLIENT_REQUEST_TIMEOUT = FlinkEnvConfiguration.FLINK_CLIENT_REQUEST_TIMEOUT().getValue().toLong(); - protected final ExecutionContext executionContext; + public final ExecutionContext executionContext; // jobId is not null only after job is submitted private JobID jobId; protected ApplicationId clusterID; @@ -91,6 +91,15 @@ public ClusterDescriptorAdapter(ExecutionContext executionContext) { /** Returns the status of the flink job. */ public JobStatus getJobStatus() throws JobExecutionException { + if (jobId == null) { + try { + LOG.info("flink getJobStatus jobId is null,sleep three seconds"); + Thread.sleep(3000); + } catch (InterruptedException e) { + + } + } + LOG.info("flink getJobStatus jobId:{}", jobId); if (jobId == null) { throw new JobExecutionException(NO_JOB_SUBMITTED.getErrorDesc()); } diff --git a/linkis-engineconn-plugins/flink/flink-core/src/main/java/org/apache/linkis/engineconnplugin/flink/client/sql/operation/OperationFactoryImpl.java b/linkis-engineconn-plugins/flink/flink-core/src/main/java/org/apache/linkis/engineconnplugin/flink/client/sql/operation/OperationFactoryImpl.java index 4329acb804..9f7d8deeff 100644 --- a/linkis-engineconn-plugins/flink/flink-core/src/main/java/org/apache/linkis/engineconnplugin/flink/client/sql/operation/OperationFactoryImpl.java +++ b/linkis-engineconn-plugins/flink/flink-core/src/main/java/org/apache/linkis/engineconnplugin/flink/client/sql/operation/OperationFactoryImpl.java @@ -49,6 +49,7 @@ public Operation createOperation(SqlCommandCall call, FlinkEngineConnContext con context, call.operands[0], Boolean.parseBoolean(call.operands[1])); break; case CREATE_TABLE: + case CREATE_FUNCTION: case DROP_TABLE: case ALTER_TABLE: case CREATE_CATALOG: diff --git a/linkis-engineconn-plugins/flink/flink-core/src/main/java/org/apache/linkis/engineconnplugin/flink/client/sql/operation/impl/DDLOperation.java b/linkis-engineconn-plugins/flink/flink-core/src/main/java/org/apache/linkis/engineconnplugin/flink/client/sql/operation/impl/DDLOperation.java index 21f6081123..ec674d5159 100644 --- a/linkis-engineconn-plugins/flink/flink-core/src/main/java/org/apache/linkis/engineconnplugin/flink/client/sql/operation/impl/DDLOperation.java +++ b/linkis-engineconn-plugins/flink/flink-core/src/main/java/org/apache/linkis/engineconnplugin/flink/client/sql/operation/impl/DDLOperation.java @@ -59,6 +59,9 @@ private String getExceptionMsg() { case CREATE_TABLE: actionMsg = "create a table"; break; + case CREATE_FUNCTION: + actionMsg = "create a function"; + break; case CREATE_DATABASE: actionMsg = "create a database"; break; diff --git a/linkis-engineconn-plugins/flink/flink-core/src/main/java/org/apache/linkis/engineconnplugin/flink/client/sql/parser/SqlCommand.java b/linkis-engineconn-plugins/flink/flink-core/src/main/java/org/apache/linkis/engineconnplugin/flink/client/sql/parser/SqlCommand.java index 9f6ef738e1..03c18c3bbd 100644 --- a/linkis-engineconn-plugins/flink/flink-core/src/main/java/org/apache/linkis/engineconnplugin/flink/client/sql/parser/SqlCommand.java +++ b/linkis-engineconn-plugins/flink/flink-core/src/main/java/org/apache/linkis/engineconnplugin/flink/client/sql/parser/SqlCommand.java @@ -46,6 +46,8 @@ public enum SqlCommand { CREATE_DATABASE, + CREATE_FUNCTION, + ALTER_DATABASE, DROP_DATABASE, diff --git a/linkis-engineconn-plugins/flink/flink-core/src/main/java/org/apache/linkis/engineconnplugin/flink/client/sql/parser/SqlCommandParserImpl.java b/linkis-engineconn-plugins/flink/flink-core/src/main/java/org/apache/linkis/engineconnplugin/flink/client/sql/parser/SqlCommandParserImpl.java index f8eb32605c..211d899bcf 100644 --- a/linkis-engineconn-plugins/flink/flink-core/src/main/java/org/apache/linkis/engineconnplugin/flink/client/sql/parser/SqlCommandParserImpl.java +++ b/linkis-engineconn-plugins/flink/flink-core/src/main/java/org/apache/linkis/engineconnplugin/flink/client/sql/parser/SqlCommandParserImpl.java @@ -152,6 +152,9 @@ private Optional parseStmt(String stmt, boolean isBlinkPlanner) } else if (node instanceof SqlCreateDatabase) { cmd = SqlCommand.CREATE_DATABASE; operands = new String[] {stmt}; + } else if (node instanceof SqlCreateFunction) { + cmd = SqlCommand.CREATE_FUNCTION; + operands = new String[] {stmt}; } else if (node instanceof SqlDropDatabase) { cmd = SqlCommand.DROP_DATABASE; operands = new String[] {stmt}; diff --git a/linkis-engineconn-plugins/flink/flink-core/src/main/java/org/apache/linkis/engineconnplugin/flink/client/utils/FlinkUdfUtils.java b/linkis-engineconn-plugins/flink/flink-core/src/main/java/org/apache/linkis/engineconnplugin/flink/client/utils/FlinkUdfUtils.java new file mode 100644 index 0000000000..e299c08f70 --- /dev/null +++ b/linkis-engineconn-plugins/flink/flink-core/src/main/java/org/apache/linkis/engineconnplugin/flink/client/utils/FlinkUdfUtils.java @@ -0,0 +1,123 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.engineconnplugin.flink.client.utils; + +import org.apache.commons.collections.CollectionUtils; +import org.apache.flink.configuration.Configuration; +import org.apache.flink.configuration.PipelineOptions; +import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; +import org.apache.flink.table.functions.UserDefinedFunction; + +import java.lang.reflect.Field; +import java.lang.reflect.Method; +import java.net.URL; +import java.net.URLClassLoader; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class FlinkUdfUtils { + + private static final Logger logger = LoggerFactory.getLogger(FlinkUdfUtils.class); + + private static final String CREATE_TEMP_FUNCTION_PATTERN = + "create\\s+temporary\\s+function\\s+(\\w+)\\s+as\\s+\"(.*?)\""; + + private static final String CREATE_TEMP_FUNCTION_SQL = + "CREATE TEMPORARY FUNCTION IF NOT EXISTS %s AS '%s' "; + + public static void addFlinkPipelineClasspaths(StreamExecutionEnvironment env, String path) { + logger.info("Flink udf start add pipeline classpaths, jar path: {}", path); + + try { + Field configuration = StreamExecutionEnvironment.class.getDeclaredField("configuration"); + configuration.setAccessible(true); + Configuration conf = (Configuration) configuration.get(env); + + Field confData = Configuration.class.getDeclaredField("confData"); + confData.setAccessible(true); + Map map = (Map) confData.get(conf); + List jarList = new ArrayList<>(); + List oldList = + conf.getOptional(PipelineOptions.CLASSPATHS).orElseGet(Collections::emptyList); + if (CollectionUtils.isNotEmpty(oldList)) { + jarList.addAll(oldList); + } + jarList.add(path); + map.put(PipelineOptions.CLASSPATHS.key(), jarList); + } catch (Exception e) { + logger.warn("Flink udf add pipeline classpaths failed", e); + } + } + + public static void loadJar(String jarPath) { + logger.info("Flink udf URLClassLoader start loadJar: {}", jarPath); + + Method method = null; + Boolean accessible = null; + try { + method = URLClassLoader.class.getDeclaredMethod("addURL", URL.class); + accessible = method.isAccessible(); + + if (accessible == false) { + method.setAccessible(true); + } + URLClassLoader classLoader = (URLClassLoader) ClassLoader.getSystemClassLoader(); + method.invoke(classLoader, new URL(jarPath)); + + } catch (Exception e) { + logger.warn("Flink udf URLClassLoader loadJar failed", e); + } finally { + if (accessible != null) { + method.setAccessible(accessible); + } + } + } + + public static String extractUdfClass(String statement) { + Pattern pattern = Pattern.compile(CREATE_TEMP_FUNCTION_PATTERN); + Matcher matcher = pattern.matcher(statement); + if (matcher.find() && matcher.groupCount() >= 2) { + return matcher.group(2); + } + return ""; + } + + public static boolean isFlinkUdf(ClassLoader classLoader, String className) { + try { + Class udfClass = classLoader.loadClass(className); + if (UserDefinedFunction.class.isAssignableFrom(udfClass)) { + return true; + } + + } catch (ClassNotFoundException e) { + logger.warn("flink udf load isFlinkUdf failed, ClassNotFoundException: {}", className); + } + return false; + } + + public static String generateFlinkUdfSql(String name, String className) { + return String.format(CREATE_TEMP_FUNCTION_SQL, name, className); + } +} diff --git a/linkis-engineconn-plugins/flink/flink-core/src/main/resources/linkis-engineconn.properties b/linkis-engineconn-plugins/flink/flink-core/src/main/resources/linkis-engineconn.properties index 2012023076..587a150eda 100644 --- a/linkis-engineconn-plugins/flink/flink-core/src/main/resources/linkis-engineconn.properties +++ b/linkis-engineconn-plugins/flink/flink-core/src/main/resources/linkis-engineconn.properties @@ -18,5 +18,5 @@ wds.linkis.server.version=v1 wds.linkis.engineconn.debug.enable=true #wds.linkis.keytab.enable=true wds.linkis.engineconn.plugin.default.class=org.apache.linkis.engineconnplugin.flink.FlinkEngineConnPlugin -wds.linkis.engine.connector.hooks=org.apache.linkis.engineconn.computation.executor.hook.ComputationEngineConnHook +wds.linkis.engine.connector.hooks=org.apache.linkis.engineconn.computation.executor.hook.ComputationEngineConnHook,org.apache.linkis.engineconnplugin.flink.hook.FlinkJarUdfEngineHook wds.linkis.engineconn.executor.manager.class=org.apache.linkis.engineconnplugin.flink.executormanager.FlinkExecutorManager \ No newline at end of file diff --git a/linkis-engineconn-plugins/flink/flink-core/src/main/scala/org/apache/linkis/engineconnplugin/flink/config/FlinkEnvConfiguration.scala b/linkis-engineconn-plugins/flink/flink-core/src/main/scala/org/apache/linkis/engineconnplugin/flink/config/FlinkEnvConfiguration.scala index 6b521dceed..a6bbceb585 100644 --- a/linkis-engineconn-plugins/flink/flink-core/src/main/scala/org/apache/linkis/engineconnplugin/flink/config/FlinkEnvConfiguration.scala +++ b/linkis-engineconn-plugins/flink/flink-core/src/main/scala/org/apache/linkis/engineconnplugin/flink/config/FlinkEnvConfiguration.scala @@ -38,7 +38,7 @@ object FlinkEnvConfiguration { val FLINK_DIST_JAR_PATH = CommonVars( "flink.dist.jar.path", - FLINK_HOME.getValue + s"/lib/flink-dist_2.11-${FLINK_VERSION.getValue}.jar" + FLINK_HOME.getValue + s"/lib/flink-dist-${FLINK_VERSION.getValue}.jar" ) val FLINK_PROVIDED_LIB_PATH = CommonVars("flink.lib.path", "") @@ -58,7 +58,9 @@ object FlinkEnvConfiguration { "The local lib path of each user in Flink EngineConn." ) - val FLINK_SHIP_DIRECTORIES = CommonVars("flink.yarn.ship-directories", "") + val FLINK_SHIP_DIRECTORIES = + CommonVars("flink.yarn.ship-directories", FLINK_HOME.getValue + "/lib") + val FLINK_SHIP_REMOTE_DIRECTORIES = CommonVars("flink.yarn.remote.ship-directories", "") val FLINK_CHECK_POINT_ENABLE = CommonVars("flink.app.checkpoint.enable", false) @@ -172,4 +174,11 @@ object FlinkEnvConfiguration { val FLINK_HANDSHAKE_WAIT_TIME_MILLS = CommonVars("linkis.flink.handshake.wait.time.mills", 60 * 1000) + val FLINK_CONF_YAML = CommonVars("flink.conf.yaml.dir", "flink-conf.yaml") + + val FLINK_YAML_MERGE_ENABLE = CommonVars("flink.yaml.merge.enable", true) + + val FLINK_ENV_JAVA_OPTS = + CommonVars("flink.env.java.opts", "env.java.opts") + } diff --git a/linkis-engineconn-plugins/flink/flink-core/src/main/scala/org/apache/linkis/engineconnplugin/flink/executor/FlinkSQLComputationExecutor.scala b/linkis-engineconn-plugins/flink/flink-core/src/main/scala/org/apache/linkis/engineconnplugin/flink/executor/FlinkSQLComputationExecutor.scala index 2e3a2bd07a..f835db9694 100644 --- a/linkis-engineconn-plugins/flink/flink-core/src/main/scala/org/apache/linkis/engineconnplugin/flink/executor/FlinkSQLComputationExecutor.scala +++ b/linkis-engineconn-plugins/flink/flink-core/src/main/scala/org/apache/linkis/engineconnplugin/flink/executor/FlinkSQLComputationExecutor.scala @@ -80,7 +80,7 @@ class FlinkSQLComputationExecutor( with FlinkExecutor { private var operation: JobOperation = _ - private var clusterDescriptor: AbstractSessionClusterDescriptorAdapter = _ + var clusterDescriptor: AbstractSessionClusterDescriptorAdapter = _ override def init(): Unit = { setCodeParser(new SQLCodeParser) @@ -236,6 +236,11 @@ class FlinkSQLComputationExecutor( super.close() } + override def tryShutdown(): Boolean = { + Utils.tryAndWarn(close()) + super.tryShutdown() + } + } class FlinkSQLStatusListener( diff --git a/linkis-engineconn-plugins/flink/flink-core/src/main/scala/org/apache/linkis/engineconnplugin/flink/factory/FlinkEngineConnFactory.scala b/linkis-engineconn-plugins/flink/flink-core/src/main/scala/org/apache/linkis/engineconnplugin/flink/factory/FlinkEngineConnFactory.scala index 1c6db3bba9..c3da253803 100644 --- a/linkis-engineconn-plugins/flink/flink-core/src/main/scala/org/apache/linkis/engineconnplugin/flink/factory/FlinkEngineConnFactory.scala +++ b/linkis-engineconn-plugins/flink/flink-core/src/main/scala/org/apache/linkis/engineconnplugin/flink/factory/FlinkEngineConnFactory.scala @@ -35,7 +35,7 @@ import org.apache.linkis.engineconnplugin.flink.config.FlinkEnvConfiguration._ import org.apache.linkis.engineconnplugin.flink.config.FlinkResourceConfiguration._ import org.apache.linkis.engineconnplugin.flink.context.{EnvironmentContext, FlinkEngineConnContext} import org.apache.linkis.engineconnplugin.flink.setting.Settings -import org.apache.linkis.engineconnplugin.flink.util.{ClassUtil, ManagerUtil} +import org.apache.linkis.engineconnplugin.flink.util.{ClassUtil, FlinkValueFormatUtil, ManagerUtil} import org.apache.linkis.governance.common.conf.GovernanceCommonConf import org.apache.linkis.manager.engineplugin.common.conf.EnvConfiguration import org.apache.linkis.manager.engineplugin.common.creation.{ @@ -55,7 +55,7 @@ import org.apache.flink.streaming.api.CheckpointingMode import org.apache.flink.streaming.api.environment.CheckpointConfig.ExternalizedCheckpointCleanup import org.apache.flink.yarn.configuration.{YarnConfigOptions, YarnDeploymentTarget} -import java.io.File +import java.io.{File, FileNotFoundException} import java.net.URL import java.text.MessageFormat import java.time.Duration @@ -63,8 +63,10 @@ import java.util import java.util.{Collections, Locale} import scala.collection.JavaConverters._ +import scala.io.Source import com.google.common.collect.{Lists, Sets} +import org.yaml.snakeyaml.Yaml class FlinkEngineConnFactory extends MultiExecutorEngineConnFactory with Logging { @@ -108,7 +110,13 @@ class FlinkEngineConnFactory extends MultiExecutorEngineConnFactory with Logging val flinkHome = FLINK_HOME.getValue(options) val flinkConfDir = FLINK_CONF_DIR.getValue(options) val flinkProvidedLibPath = FLINK_PROVIDED_LIB_PATH.getValue(options) - val flinkDistJarPath = FLINK_DIST_JAR_PATH.getValue(options) + val flinkVersion = FlinkEnvConfiguration.FLINK_VERSION.getValue(options) + var flinkDistJarPath = FLINK_DIST_JAR_PATH.getValue(options) + if ( + StringUtils.isNotBlank(flinkVersion) && flinkVersion.equalsIgnoreCase(FLINK_1_12_2_VERSION) + ) { + flinkDistJarPath = flinkDistJarPath.replaceFirst("flink-dist", "flink-dist_2.11") + } // Local lib path val providedLibDirsArray = FLINK_LIB_LOCAL_PATH.getValue(options).split(",") // Ship directories @@ -126,7 +134,6 @@ class FlinkEngineConnFactory extends MultiExecutorEngineConnFactory with Logging ) } otherParams.put(GovernanceCommonConf.EC_APP_MANAGE_MODE.key, flinkClientType.toLowerCase()) - val flinkVersion = FlinkEnvConfiguration.FLINK_VERSION.getValue(options) FlinkVersionThreadLocal.setFlinkVersion(flinkVersion) val context = new EnvironmentContext( defaultEnv, @@ -191,7 +198,15 @@ class FlinkEngineConnFactory extends MultiExecutorEngineConnFactory with Logging flinkConfig.setInteger(TaskManagerOptions.NUM_TASK_SLOTS, numberOfTaskSlots) // set extra configs options.asScala.filter { case (key, _) => key.startsWith(FLINK_CONFIG_PREFIX) }.foreach { - case (key, value) => flinkConfig.setString(key.substring(FLINK_CONFIG_PREFIX.length), value) + case (key, value) => + var flinkConfigValue = value + if ( + FlinkEnvConfiguration.FLINK_YAML_MERGE_ENABLE.getValue && key + .equals(FLINK_CONFIG_PREFIX + FLINK_ENV_JAVA_OPTS.getValue) + ) { + flinkConfigValue = getExtractJavaOpts(value) + } + flinkConfig.setString(key.substring(FLINK_CONFIG_PREFIX.length), flinkConfigValue) } // set kerberos config if (FLINK_KERBEROS_ENABLE.getValue(options)) { @@ -290,6 +305,44 @@ class FlinkEngineConnFactory extends MultiExecutorEngineConnFactory with Logging context } + private def getExtractJavaOpts(envJavaOpts: String): String = { + var defaultJavaOpts = "" + val yamlFilePath = FLINK_CONF_DIR.getValue + val yamlFile = yamlFilePath + "/" + FLINK_CONF_YAML.getHotValue() + if (new File(yamlFile).exists()) { + val source = Source.fromFile(yamlFile) + try { + val yamlContent = source.mkString + val yaml = new Yaml() + val configMap = yaml.loadAs(yamlContent, classOf[util.LinkedHashMap[String, Object]]) + if (configMap.containsKey(FLINK_ENV_JAVA_OPTS.getValue)) { + defaultJavaOpts = configMap.get(FLINK_ENV_JAVA_OPTS.getValue).toString + } + } finally { + source.close() + } + } else { + val inputStream = getClass.getResourceAsStream(yamlFile) + if (inputStream != null) { + val source = Source.fromInputStream(inputStream) + try { + val yamlContent = source.mkString + val yaml = new Yaml() + val configMap = yaml.loadAs(yamlContent, classOf[util.LinkedHashMap[String, Object]]) + if (configMap.containsKey(FLINK_ENV_JAVA_OPTS.getValue)) { + defaultJavaOpts = configMap.get(FLINK_ENV_JAVA_OPTS.getValue).toString + } + } finally { + source.close() + } + } else { + throw new FileNotFoundException("YAML file not found in both file system and classpath.") + } + } + val merged = FlinkValueFormatUtil.mergeAndDeduplicate(defaultJavaOpts, envJavaOpts) + merged + } + protected def isOnceEngineConn(labels: util.List[Label[_]]): Boolean = { val engineConnModeLabel = getEngineConnModeLabel(labels) engineConnModeLabel != null && (EngineConnMode.toEngineConnMode( diff --git a/linkis-engineconn-plugins/flink/flink-core/src/main/scala/org/apache/linkis/engineconnplugin/flink/hook/FlinkJarUdfEngineHook.scala b/linkis-engineconn-plugins/flink/flink-core/src/main/scala/org/apache/linkis/engineconnplugin/flink/hook/FlinkJarUdfEngineHook.scala new file mode 100644 index 0000000000..bc3d0f1f4a --- /dev/null +++ b/linkis-engineconn-plugins/flink/flink-core/src/main/scala/org/apache/linkis/engineconnplugin/flink/hook/FlinkJarUdfEngineHook.scala @@ -0,0 +1,107 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.engineconnplugin.flink.hook + +import org.apache.linkis.engineconn.common.creation.EngineCreationContext +import org.apache.linkis.engineconn.common.engineconn.EngineConn +import org.apache.linkis.engineconn.computation.executor.hook.UDFLoadEngineConnHook +import org.apache.linkis.engineconn.core.executor.ExecutorManager +import org.apache.linkis.engineconnplugin.flink.client.utils.FlinkUdfUtils +import org.apache.linkis.engineconnplugin.flink.executor.FlinkSQLComputationExecutor +import org.apache.linkis.manager.label.entity.Label +import org.apache.linkis.manager.label.entity.engine.{CodeLanguageLabel, EngineTypeLabel, RunType} +import org.apache.linkis.udf.utils.ConstantVar +import org.apache.linkis.udf.vo.UDFInfoVo + +import org.apache.commons.lang3.StringUtils + +import scala.collection.JavaConverters.asScalaBufferConverter + +class FlinkJarUdfEngineHook extends UDFLoadEngineConnHook { + override val udfType: BigInt = ConstantVar.UDF_JAR + override val category: String = ConstantVar.UDF + override val runType = RunType.SQL + + var labels: Array[Label[_]] = null + + override protected def constructCode(udfInfo: UDFInfoVo): String = { + val path: String = udfInfo.getPath + val registerFormat: String = udfInfo.getRegisterFormat + + if (StringUtils.isBlank(path) && StringUtils.isBlank(registerFormat)) { + logger.warn("Flink udfInfo path or registerFormat cannot is empty") + return "" + } + + val udfClassName: String = FlinkUdfUtils.extractUdfClass(registerFormat) + if (StringUtils.isBlank(udfClassName)) { + logger.warn("Flink extract udf class name cannot is empty") + return "" + } + + FlinkUdfUtils.loadJar(path) + + if (!FlinkUdfUtils.isFlinkUdf(ClassLoader.getSystemClassLoader(), udfClassName)) { + logger.warn( + "There is no extends Flink UserDefinedFunction, skip loading flink udf: {} ", + path + ) + return "" + } + + val flinkUdfSql: String = + FlinkUdfUtils.generateFlinkUdfSql(udfInfo.getUdfName, udfClassName) + + logger.info( + s"Flink start load udf, udfName:${udfInfo.getUdfName}, udfJar:${path}, udfClass:${udfClassName}\n" + ) + + if (labels != null && labels.nonEmpty) { + val executor = ExecutorManager.getInstance.getExecutorByLabels(labels) + executor match { + case computationExecutor: FlinkSQLComputationExecutor => + FlinkUdfUtils.addFlinkPipelineClasspaths( + computationExecutor.clusterDescriptor.executionContext.getStreamExecutionEnvironment, + path + ) + case _ => + } + } + + "%sql\n" + flinkUdfSql + } + + override def afterExecutionExecute( + engineCreationContext: EngineCreationContext, + engineConn: EngineConn + ): Unit = { + val codeLanguageLabel = new CodeLanguageLabel + engineCreationContext.getLabels().asScala.find(_.isInstanceOf[EngineTypeLabel]) match { + case Some(engineTypeLabel) => + codeLanguageLabel.setCodeType( + getRealRunType(engineTypeLabel.asInstanceOf[EngineTypeLabel].getEngineType).toString + ) + case None => + codeLanguageLabel.setCodeType(runType.toString) + } + labels = Array[Label[_]](codeLanguageLabel) + + super.afterExecutionExecute(engineCreationContext, engineConn) + } + +} diff --git a/linkis-engineconn-plugins/flink/flink-core/src/main/scala/org/apache/linkis/engineconnplugin/flink/launch/FlinkEngineConnLaunchBuilder.scala b/linkis-engineconn-plugins/flink/flink-core/src/main/scala/org/apache/linkis/engineconnplugin/flink/launch/FlinkEngineConnLaunchBuilder.scala index cbcae4ea54..13a5bae4d5 100644 --- a/linkis-engineconn-plugins/flink/flink-core/src/main/scala/org/apache/linkis/engineconnplugin/flink/launch/FlinkEngineConnLaunchBuilder.scala +++ b/linkis-engineconn-plugins/flink/flink-core/src/main/scala/org/apache/linkis/engineconnplugin/flink/launch/FlinkEngineConnLaunchBuilder.scala @@ -24,14 +24,28 @@ import org.apache.linkis.hadoop.common.conf.HadoopConf import org.apache.linkis.manager.common.protocol.bml.BmlResource import org.apache.linkis.manager.engineplugin.common.conf.EnvConfiguration import org.apache.linkis.manager.engineplugin.common.launch.entity.EngineConnBuildRequest -import org.apache.linkis.manager.engineplugin.common.launch.process.Environment.{variable, USER} -import org.apache.linkis.manager.engineplugin.common.launch.process.JavaProcessEngineConnLaunchBuilder -import org.apache.linkis.manager.label.entity.engine.UserCreatorLabel +import org.apache.linkis.manager.engineplugin.common.launch.process.{ + Environment, + JavaProcessEngineConnLaunchBuilder +} +import org.apache.linkis.manager.engineplugin.common.launch.process.Environment.{ + variable, + PWD, + USER +} +import org.apache.linkis.manager.engineplugin.common.launch.process.LaunchConstants.{ + addPathToClassPath, + CLASS_PATH_SEPARATOR +} +import org.apache.linkis.manager.label.entity.engine.{EngineConnMode, UserCreatorLabel} +import org.apache.linkis.manager.label.utils.LabelUtil import java.util import scala.collection.JavaConverters._ +import com.google.common.collect.Lists + class FlinkEngineConnLaunchBuilder extends JavaProcessEngineConnLaunchBuilder { override protected def getCommands(implicit @@ -82,6 +96,19 @@ class FlinkEngineConnLaunchBuilder extends JavaProcessEngineConnLaunchBuilder { bmlResources } + override def getEnvironment(implicit + engineConnBuildRequest: EngineConnBuildRequest + ): util.Map[String, String] = { + val environment = new util.HashMap[String, String] + addPathToClassPath(environment, variable(PWD)) + val linkisEnvironment = super.getEnvironment + val linkisClassPath = linkisEnvironment.get(Environment.CLASSPATH.toString) + val v = environment.get(Environment.CLASSPATH.toString) + CLASS_PATH_SEPARATOR + linkisClassPath + environment.put(Environment.CLASSPATH.toString, v) + logger.info(environment.asScala.map(e => s"${e._1}->${e._2}").mkString(",")) + environment + } + private def contentToBmlResource(userName: String, content: String): BmlResource = { val contentMap = JsonUtils.jackson.readValue(content, classOf[util.Map[String, Object]]) contentToBmlResource(userName, contentMap) @@ -112,4 +139,17 @@ class FlinkEngineConnLaunchBuilder extends JavaProcessEngineConnLaunchBuilder { override protected def ifAddHiveConfigPath: Boolean = true + override protected def getEngineConnManagerHooks(implicit + engineConnBuildRequest: EngineConnBuildRequest + ): java.util.List[String] = if (isOnceMode) { + super.getEngineConnManagerHooks(engineConnBuildRequest) + } else { + Lists.newArrayList("JarUDFLoadECMHook") + } + + def isOnceMode: Boolean = { + val engineConnMode = LabelUtil.getEngineConnMode(engineConnBuildRequest.labels) + EngineConnMode.toEngineConnMode(engineConnMode) == EngineConnMode.Once + } + } diff --git a/linkis-engineconn-plugins/flink/flink-core/src/main/scala/org/apache/linkis/engineconnplugin/flink/util/FlinkValueFormatUtil.scala b/linkis-engineconn-plugins/flink/flink-core/src/main/scala/org/apache/linkis/engineconnplugin/flink/util/FlinkValueFormatUtil.scala index 62782507eb..0160e97eab 100644 --- a/linkis-engineconn-plugins/flink/flink-core/src/main/scala/org/apache/linkis/engineconnplugin/flink/util/FlinkValueFormatUtil.scala +++ b/linkis-engineconn-plugins/flink/flink-core/src/main/scala/org/apache/linkis/engineconnplugin/flink/util/FlinkValueFormatUtil.scala @@ -36,4 +36,61 @@ object FlinkValueFormatUtil { case _ => null } + def mergeAndDeduplicate(defaultJavaOpts: String, envJavaOpts: String): String = { + val patternX = """-XX:([^\s]+)=([^\s]+)""".r + val keyValueMapX = patternX + .findAllMatchIn(envJavaOpts) + .map { matchResult => + val key = matchResult.group(1) + val value = matchResult.group(2) + (key, value) + } + .toMap + + val patternD = """-D([^\s]+)=([^\s]+)""".r + val keyValueMapD = patternD + .findAllMatchIn(envJavaOpts) + .map { matchResult => + val key = matchResult.group(1) + val value = matchResult.group(2) + (key, value) + } + .toMap + val xloggcPattern = """-Xloggc:[^\s]+""".r + val xloggcValueStr1 = xloggcPattern.findFirstMatchIn(defaultJavaOpts).getOrElse("").toString + val xloggcValueStr2 = xloggcPattern.findFirstMatchIn(envJavaOpts).getOrElse("").toString + var escapedXloggcValue = "" + var replaceStr1 = "" + var replaceStr2 = "" + if (xloggcValueStr1.nonEmpty && xloggcValueStr2.nonEmpty) { + escapedXloggcValue = xloggcValueStr2.replace("\\<", "<").replace("\\>", ">") + replaceStr1 = defaultJavaOpts.replace(xloggcValueStr1, escapedXloggcValue) + replaceStr2 = envJavaOpts.replace(xloggcValueStr2, "") + } + if (xloggcValueStr1.nonEmpty && xloggcValueStr2.isEmpty) { + escapedXloggcValue = xloggcValueStr1.replace("\\<", "<").replace("\\>", ">") + replaceStr1 = defaultJavaOpts.replace(xloggcValueStr1, escapedXloggcValue) + replaceStr2 = envJavaOpts + } + if (xloggcValueStr1.isEmpty && xloggcValueStr2.isEmpty) { + replaceStr1 = defaultJavaOpts + replaceStr2 = envJavaOpts + } + val MergedStringX = keyValueMapX.foldLeft(replaceStr1) { (result, entry) => + val (key, value) = entry + val oldValue = s"$key=[^\\s]+" + val newValue = key + "=" + value + result.replaceAll(oldValue, newValue) + } + + val MergedStringD = keyValueMapD.foldLeft(MergedStringX) { (result, entry) => + val (key, value) = entry + val oldValue = s"$key=[^\\s]+" + val newValue = key + "=" + value + result.replaceAll(oldValue, newValue) + } + val javaOpts = (MergedStringD.split("\\s+") ++ replaceStr2.split("\\s+")).distinct.mkString(" ") + javaOpts + } + } diff --git a/linkis-engineconn-plugins/flink/flink-core/src/test/java/org/apache/linkis/engineplugin/flink/LinkisFlinkUdfExample.java b/linkis-engineconn-plugins/flink/flink-core/src/test/java/org/apache/linkis/engineplugin/flink/LinkisFlinkUdfExample.java new file mode 100644 index 0000000000..ce2b05e693 --- /dev/null +++ b/linkis-engineconn-plugins/flink/flink-core/src/test/java/org/apache/linkis/engineplugin/flink/LinkisFlinkUdfExample.java @@ -0,0 +1,26 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.engineplugin.flink; + +import org.apache.flink.table.functions.ScalarFunction; + +public class LinkisFlinkUdfExample extends ScalarFunction { + public String eval(String str) { + return String.format("linkis flink udf test: %s", str); + } +} diff --git a/linkis-engineconn-plugins/hbase/hbase-core/src/main/java/org/apache/linkis/manager/engineplugin/hbase/shell/HBaseShellSession.java b/linkis-engineconn-plugins/hbase/hbase-core/src/main/java/org/apache/linkis/manager/engineplugin/hbase/shell/HBaseShellSession.java index a0aef55797..af61eab186 100644 --- a/linkis-engineconn-plugins/hbase/hbase-core/src/main/java/org/apache/linkis/manager/engineplugin/hbase/shell/HBaseShellSession.java +++ b/linkis-engineconn-plugins/hbase/hbase-core/src/main/java/org/apache/linkis/manager/engineplugin/hbase/shell/HBaseShellSession.java @@ -271,6 +271,9 @@ private Result executeCmd(String cmd) { if (StringUtils.isBlank(res) && o != null) { res = o.toString(); } + if (res.contains("ERROR: ") && res.contains("For usage try 'help")) { + return Result.failed(getStackTrace(new Throwable(new Exception(res))), new Exception(res)); + } return Result.ok(res); } catch (Exception e) { return Result.failed(getStackTrace(e), e); diff --git a/linkis-engineconn-plugins/hbase/hbase-core/src/main/resources/linkis-engineconn.properties b/linkis-engineconn-plugins/hbase/hbase-core/src/main/resources/linkis-engineconn.properties index a3f4a74de3..674d7be02e 100644 --- a/linkis-engineconn-plugins/hbase/hbase-core/src/main/resources/linkis-engineconn.properties +++ b/linkis-engineconn-plugins/hbase/hbase-core/src/main/resources/linkis-engineconn.properties @@ -21,4 +21,4 @@ wds.linkis.engineconn.plugin.default.class=org.apache.linkis.manager.engineplugi #wds.linkis.engine.io.opts=" -Dfile.encoding=UTF-8 -agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=49100 " wds.linkis.engineconn.support.parallelism=true wds.linkis.rpc.cache.expire.time=0 -wds.linkis.engineconn.max.free.time=0 \ No newline at end of file +#wds.linkis.engineconn.max.free.time=0 \ No newline at end of file diff --git a/linkis-engineconn-plugins/hive/pom.xml b/linkis-engineconn-plugins/hive/pom.xml index b79ab7be1f..28ae9dd1ce 100644 --- a/linkis-engineconn-plugins/hive/pom.xml +++ b/linkis-engineconn-plugins/hive/pom.xml @@ -396,5 +396,44 @@ + + + + eureka + + true + + discovery + eureka + + + + + + org.antlr + antlr-runtime + provided + + + + + + nacos + + + discovery + nacos + + + + + + org.antlr + antlr-runtime + compile + + + + diff --git a/linkis-engineconn-plugins/hive/src/main/assembly/distribution.xml b/linkis-engineconn-plugins/hive/src/main/assembly/distribution.xml index 667587cffc..bd9d705013 100644 --- a/linkis-engineconn-plugins/hive/src/main/assembly/distribution.xml +++ b/linkis-engineconn-plugins/hive/src/main/assembly/distribution.xml @@ -104,7 +104,6 @@ javax.xml.bind:jaxb-api:jar javax.xml.stream:stax-api:jar mysql:mysql-connector-java:jar - org.antlr:antlr-runtime:jar org.antlr:stringtemplate:jar org.apache.commons:commons-compress:jar org.apache.commons:commons-math:jar diff --git a/linkis-engineconn-plugins/impala/src/main/scala/org/apache/linkis/engineplugin/impala/executor/ImpalaEngineConnExecutor.scala b/linkis-engineconn-plugins/impala/src/main/scala/org/apache/linkis/engineplugin/impala/executor/ImpalaEngineConnExecutor.scala index 23cd1a0e6f..97613f3f94 100644 --- a/linkis-engineconn-plugins/impala/src/main/scala/org/apache/linkis/engineplugin/impala/executor/ImpalaEngineConnExecutor.scala +++ b/linkis-engineconn-plugins/impala/src/main/scala/org/apache/linkis/engineplugin/impala/executor/ImpalaEngineConnExecutor.scala @@ -17,43 +17,72 @@ package org.apache.linkis.engineplugin.impala.executor -import org.apache.commons.collections.MapUtils -import org.apache.commons.io.IOUtils -import org.apache.commons.lang3.StringUtils -import org.apache.commons.lang3.exception.ExceptionUtils import org.apache.linkis.common.log.LogUtils import org.apache.linkis.common.utils.{OverloadUtils, Utils} -import org.apache.linkis.engineconn.common.password.{CommandPasswordCallback, StaticPasswordCallback} -import org.apache.linkis.engineconn.computation.executor.execute.{ConcurrentComputationExecutor, EngineExecutionContext} +import org.apache.linkis.engineconn.common.password.{ + CommandPasswordCallback, + StaticPasswordCallback +} +import org.apache.linkis.engineconn.computation.executor.execute.{ + ConcurrentComputationExecutor, + EngineExecutionContext +} import org.apache.linkis.engineconn.core.EngineConnObject +import org.apache.linkis.engineplugin.impala.client.{ + ExecutionListener, + ImpalaClient, + ImpalaResultSet +} import org.apache.linkis.engineplugin.impala.client.ImpalaResultSet.Row -import org.apache.linkis.engineplugin.impala.client.exception.{ImpalaEngineException, ImpalaErrorCodeSummary} +import org.apache.linkis.engineplugin.impala.client.exception.{ + ImpalaEngineException, + ImpalaErrorCodeSummary +} import org.apache.linkis.engineplugin.impala.client.protocol.{ExecProgress, ExecStatus} -import org.apache.linkis.engineplugin.impala.client.thrift.{ImpalaThriftClient, ImpalaThriftSessionFactory} -import org.apache.linkis.engineplugin.impala.client.{ExecutionListener, ImpalaClient, ImpalaResultSet} +import org.apache.linkis.engineplugin.impala.client.thrift.{ + ImpalaThriftClient, + ImpalaThriftSessionFactory +} import org.apache.linkis.engineplugin.impala.conf.ImpalaConfiguration._ import org.apache.linkis.engineplugin.impala.conf.ImpalaEngineConfig import org.apache.linkis.governance.common.paser.SQLCodeParser -import org.apache.linkis.manager.common.entity.resource.{CommonNodeResource, LoadResource, NodeResource} +import org.apache.linkis.manager.common.entity.resource.{ + CommonNodeResource, + LoadResource, + NodeResource +} import org.apache.linkis.manager.engineplugin.common.util.NodeResourceUtils import org.apache.linkis.manager.label.entity.Label import org.apache.linkis.manager.label.entity.engine.{EngineTypeLabel, UserCreatorLabel} import org.apache.linkis.protocol.engine.JobProgressInfo import org.apache.linkis.rpc.Sender -import org.apache.linkis.scheduler.executer.{CompletedExecuteResponse, ErrorExecuteResponse, ExecuteResponse, SuccessExecuteResponse} +import org.apache.linkis.scheduler.executer.{ + CompletedExecuteResponse, + ErrorExecuteResponse, + ExecuteResponse, + SuccessExecuteResponse +} import org.apache.linkis.storage.domain.Column import org.apache.linkis.storage.resultset.ResultSetFactory import org.apache.linkis.storage.resultset.table.{TableMetaData, TableRecord} + +import org.apache.commons.collections.MapUtils +import org.apache.commons.io.IOUtils +import org.apache.commons.lang3.StringUtils +import org.apache.commons.lang3.exception.ExceptionUtils + import org.springframework.util.CollectionUtils +import javax.net.SocketFactory +import javax.net.ssl._ +import javax.security.auth.callback.{Callback, CallbackHandler, NameCallback, PasswordCallback} + import java.io.FileInputStream import java.security.KeyStore import java.util import java.util.concurrent.ConcurrentHashMap import java.util.function.Consumer -import javax.net.SocketFactory -import javax.net.ssl._ -import javax.security.auth.callback.{Callback, CallbackHandler, NameCallback, PasswordCallback} + import scala.collection.JavaConverters._ class ImpalaEngineConnExecutor(override val outputPrintLimit: Int, val id: Int) diff --git a/linkis-engineconn-plugins/io_file/src/test/java/executor/IoEngineConnExecutorTest.java b/linkis-engineconn-plugins/io_file/src/test/java/executor/IoEngineConnExecutorTest.java new file mode 100644 index 0000000000..252ec95ab5 --- /dev/null +++ b/linkis-engineconn-plugins/io_file/src/test/java/executor/IoEngineConnExecutorTest.java @@ -0,0 +1,77 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package executor; + +import org.apache.linkis.common.io.FsPath; +import org.apache.linkis.engineconn.computation.executor.execute.EngineExecutionContext; +import org.apache.linkis.manager.engineplugin.io.executor.IoEngineConnExecutor; +import org.apache.linkis.scheduler.executer.AliasOutputExecuteResponse; +import org.apache.linkis.scheduler.executer.ExecuteResponse; +import org.apache.linkis.storage.domain.MethodEntity; +import org.apache.linkis.storage.domain.MethodEntitySerializer; + +import org.assertj.core.api.Assertions; +import org.junit.jupiter.api.Test; + +class IoEngineConnExecutorTest { + + @Test + public void testExecuteLine() { + // test init + IoEngineConnExecutor ioEngineConnExecutor = new IoEngineConnExecutor(1, Integer.MAX_VALUE); + EngineExecutionContext engineExecutionContext = + new EngineExecutionContext(ioEngineConnExecutor, "hadoop"); + engineExecutionContext.setJobId("jobId-1"); + Object[] objects = new Object[10]; + MethodEntity methodEntity = + new MethodEntity(0L, "file", "hadoop", "hadoop", "localhost", "init", objects); + AliasOutputExecuteResponse executeResponse = + (AliasOutputExecuteResponse) + ioEngineConnExecutor.executeLine( + engineExecutionContext, MethodEntitySerializer.serializer(methodEntity)); + Assertions.assertThat(executeResponse).isNotNull(); + Assertions.assertThat(executeResponse.alias()).isEqualTo("0"); + + // test write + String filePath = this.getClass().getResource("/testIoResult.dolphin").getFile().toString(); + FsPath fsPath = new FsPath(filePath); + String fsPathStr = MethodEntitySerializer.serializerJavaObject(fsPath); + objects = new Object[3]; + objects[0] = fsPathStr; + objects[1] = true; + objects[2] = "dolphin000000000300000000040,110000000016aGVsbG8gd29ybGQ="; + methodEntity = new MethodEntity(0L, "file", "hadoop", "hadoop", "localhost", "write", objects); + ExecuteResponse writeResponse = + ioEngineConnExecutor.executeLine( + engineExecutionContext, MethodEntitySerializer.serializer(methodEntity)); + System.out.println(writeResponse); + Assertions.assertThat(executeResponse).isNotNull(); + + // test read + objects = new Object[1]; + objects[0] = fsPathStr; + methodEntity = new MethodEntity(0L, "file", "hadoop", "hadoop", "localhost", "read", objects); + AliasOutputExecuteResponse readResponse = + (AliasOutputExecuteResponse) + ioEngineConnExecutor.executeLine( + engineExecutionContext, MethodEntitySerializer.serializer(methodEntity)); + Assertions.assertThat(readResponse).isNotNull(); + Assertions.assertThat(readResponse.output()) + .isEqualTo("dolphin000000000300000000040,110000000016aGVsbG8gd29ybGQ="); + } +} diff --git a/linkis-engineconn-plugins/io_file/src/test/scala/org/apache/linkis/manager/engineplugin/io/executor/IoEngineConnExecutorTest.java b/linkis-engineconn-plugins/io_file/src/test/scala/org/apache/linkis/manager/engineplugin/io/executor/IoEngineConnExecutorTest.java deleted file mode 100644 index 5dcc883114..0000000000 --- a/linkis-engineconn-plugins/io_file/src/test/scala/org/apache/linkis/manager/engineplugin/io/executor/IoEngineConnExecutorTest.java +++ /dev/null @@ -1,73 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.manager.engineplugin.io.executor; - -import org.apache.linkis.common.io.FsPath; -import org.apache.linkis.engineconn.computation.executor.execute.ComputationExecutor; -import org.apache.linkis.engineconn.computation.executor.execute.EngineExecutionContext; -import org.apache.linkis.manager.engineplugin.io.conf.IOEngineConnConfiguration; -import org.apache.linkis.manager.engineplugin.io.factory.IoEngineConnFactory; -import org.apache.linkis.scheduler.executer.AliasOutputExecuteResponse; -import org.apache.linkis.scheduler.executer.ExecuteResponse; -import org.apache.linkis.storage.domain.MethodEntity; -import org.apache.linkis.storage.domain.MethodEntitySerializer; -import org.assertj.core.api.Assertions; -import org.junit.jupiter.api.Test; - - - -class IoEngineConnExecutorTest { - - @Test - public void testExecuteLine() { - //test init - IoEngineConnExecutor ioEngineConnExecutor = new IoEngineConnExecutor(1, Integer.MAX_VALUE); - EngineExecutionContext engineExecutionContext = new EngineExecutionContext(ioEngineConnExecutor, "hadoop"); - engineExecutionContext.setJobId("jobId-1"); - Object[] objects = new Object[10]; - MethodEntity methodEntity = new MethodEntity(0L, "file", "hadoop", "hadoop", - "localhost", "init", objects); - AliasOutputExecuteResponse executeResponse = (AliasOutputExecuteResponse)ioEngineConnExecutor.executeLine(engineExecutionContext, MethodEntitySerializer.serializer(methodEntity)); - Assertions.assertThat(executeResponse).isNotNull(); - Assertions.assertThat(executeResponse.alias()).isEqualTo("0"); - - //test write - String filePath = this.getClass().getResource("/testIoResult.dolphin").getFile().toString(); - FsPath fsPath = new FsPath(filePath); - String fsPathStr = MethodEntitySerializer.serializerJavaObject(fsPath); - objects = new Object[3]; - objects[0] = fsPathStr; - objects[1] = true; - objects[2] = "dolphin000000000300000000040,110000000016aGVsbG8gd29ybGQ="; - methodEntity = new MethodEntity(0L, "file", "hadoop", "hadoop", - "localhost", "write", objects); - ExecuteResponse writeResponse = ioEngineConnExecutor.executeLine(engineExecutionContext, MethodEntitySerializer.serializer(methodEntity)); - System.out.println(writeResponse); - Assertions.assertThat(executeResponse).isNotNull(); - - //test read - objects = new Object[1]; - objects[0] = fsPathStr; - methodEntity = new MethodEntity(0L, "file", "hadoop", "hadoop", - "localhost", "read", objects); - AliasOutputExecuteResponse readResponse = (AliasOutputExecuteResponse)ioEngineConnExecutor.executeLine(engineExecutionContext, MethodEntitySerializer.serializer(methodEntity)); - Assertions.assertThat(readResponse).isNotNull(); - Assertions.assertThat(readResponse.output()).isEqualTo("dolphin000000000300000000040,110000000016aGVsbG8gd29ybGQ="); - } - -} \ No newline at end of file diff --git a/linkis-engineconn-plugins/jdbc/src/main/java/org/apache/linkis/manager/engineplugin/jdbc/ConnectionManager.java b/linkis-engineconn-plugins/jdbc/src/main/java/org/apache/linkis/manager/engineplugin/jdbc/ConnectionManager.java index b9cd479457..a49613f8d1 100644 --- a/linkis-engineconn-plugins/jdbc/src/main/java/org/apache/linkis/manager/engineplugin/jdbc/ConnectionManager.java +++ b/linkis-engineconn-plugins/jdbc/src/main/java/org/apache/linkis/manager/engineplugin/jdbc/ConnectionManager.java @@ -23,16 +23,19 @@ import org.apache.linkis.manager.engineplugin.jdbc.exception.JDBCParamsIllegalException; import org.apache.linkis.manager.engineplugin.jdbc.utils.JdbcParamUtils; -import org.apache.commons.dbcp.BasicDataSource; import org.apache.commons.lang3.StringUtils; import org.apache.hadoop.security.UserGroupInformation; import javax.sql.DataSource; +import java.io.Closeable; import java.security.PrivilegedExceptionAction; -import java.sql.*; +import java.sql.Connection; +import java.sql.SQLException; +import java.sql.Statement; import java.text.MessageFormat; -import java.util.*; +import java.util.HashMap; +import java.util.Map; import java.util.concurrent.Callable; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; @@ -42,7 +45,8 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import static org.apache.linkis.manager.engineplugin.jdbc.JdbcAuthType.*; +import static org.apache.linkis.manager.engineplugin.jdbc.JdbcAuthType.USERNAME; +import static org.apache.linkis.manager.engineplugin.jdbc.JdbcAuthType.of; import static org.apache.linkis.manager.engineplugin.jdbc.errorcode.JDBCErrorCodeSummary.*; public class ConnectionManager { @@ -103,8 +107,10 @@ public void close() { } for (DataSource dataSource : this.dataSourceFactories.values()) { try { - ((BasicDataSource) dataSource).close(); - } catch (SQLException e) { + if (dataSource instanceof Closeable) { + ((Closeable) dataSource).close(); + } + } catch (Exception e) { LOG.error("Error while closing datasource...", e); } } diff --git a/linkis-engineconn-plugins/jdbc/src/main/scala/org/apache/linkis/manager/engineplugin/jdbc/executor/JDBCEngineConnExecutor.scala b/linkis-engineconn-plugins/jdbc/src/main/scala/org/apache/linkis/manager/engineplugin/jdbc/executor/JDBCEngineConnExecutor.scala index 336d1197f7..8a2d64fa76 100644 --- a/linkis-engineconn-plugins/jdbc/src/main/scala/org/apache/linkis/manager/engineplugin/jdbc/executor/JDBCEngineConnExecutor.scala +++ b/linkis-engineconn-plugins/jdbc/src/main/scala/org/apache/linkis/manager/engineplugin/jdbc/executor/JDBCEngineConnExecutor.scala @@ -19,6 +19,7 @@ package org.apache.linkis.manager.engineplugin.jdbc.executor import org.apache.linkis.common.conf.Configuration import org.apache.linkis.common.utils.{OverloadUtils, Utils} +import org.apache.linkis.engineconn.computation.executor.entity.EngineConnTask import org.apache.linkis.engineconn.computation.executor.execute.{ ConcurrentComputationExecutor, EngineExecutionContext @@ -78,6 +79,8 @@ class JDBCEngineConnExecutor(override val outputPrintLimit: Int, val id: Int) private val progressMonitors: util.Map[String, ProgressMonitor[_]] = new ConcurrentHashMap[String, ProgressMonitor[_]]() + private val connectionCache: util.Map[String, Connection] = new util.HashMap[String, Connection]() + override def init(): Unit = { logger.info("jdbc executor start init.") setCodeParser(new SQLCodeParser) @@ -87,49 +90,59 @@ class JDBCEngineConnExecutor(override val outputPrintLimit: Int, val id: Int) } } - override def executeLine( - engineExecutorContext: EngineExecutionContext, - code: String - ): ExecuteResponse = { - val realCode = code.trim() - val taskId = engineExecutorContext.getJobId.get + override def execute(engineConnTask: EngineConnTask): ExecuteResponse = { + val executeResponse = super.execute(engineConnTask) + if (StringUtils.isNotBlank(engineConnTask.getTaskId)) { + val connection = connectionCache.remove(engineConnTask.getTaskId) + logger.info(s"remove task ${engineConnTask.getTaskId} connection") + Utils.tryAndWarn(connection.close()) + } + executeResponse + } - var properties: util.Map[String, String] = Collections.emptyMap() + private def getConnection(engineExecutorContext: EngineExecutionContext): Connection = { - Utils.tryCatch({ - properties = getJDBCRuntimeParams(engineExecutorContext) - }) { e: Throwable => - logger.error(s"try to build JDBC runtime params error! $e") - return ErrorExecuteResponse(e.getMessage, e) + val taskId = engineExecutorContext.getJobId.orNull + if (StringUtils.isNotBlank(taskId) && connectionCache.containsKey(taskId)) { + logger.info( + s"Task ${taskId} paragraph ${engineExecutorContext.getCurrentParagraph} from cache get connection" + ) + return connectionCache.get(taskId) } - + val properties: util.Map[String, String] = getJDBCRuntimeParams(engineExecutorContext) logger.info(s"The jdbc properties is: $properties") val dataSourceName = properties.get(JDBCEngineConnConstant.JDBC_ENGINE_RUN_TIME_DS) val dataSourceMaxVersionId = properties.get(JDBCEngineConnConstant.JDBC_ENGINE_RUN_TIME_DS_MAX_VERSION_ID) logger.info( - s"The data source name is [$dataSourceName], and the jdbc client begins to run jdbc code:\n ${realCode.trim}" + s"The data source name is [$dataSourceName], and the jdbc client begins to run task ${taskId}" ) - var connection: Connection = null - var statement: Statement = null - var resultSet: ResultSet = null logger.info(s"The data source properties is $properties") - Utils.tryCatch({ - /* url + user as the cache key */ - val jdbcUrl: String = properties.get(JDBCEngineConnConstant.JDBC_URL) - val execUser: String = properties.get(JDBCEngineConnConstant.JDBC_SCRIPTS_EXEC_USER) - val proxyUser: String = properties.get(JDBCEngineConnConstant.JDBC_PROXY_USER_PROPERTY) - var dataSourceIdentifier = s"$jdbcUrl-$execUser-$proxyUser" - /* If datasource is used, use datasource name as the cache key */ - if (StringUtils.isNotBlank(dataSourceName)) { - dataSourceIdentifier = s"$dataSourceName-$dataSourceMaxVersionId" - } - connection = connectionManager.getConnection(dataSourceIdentifier, properties) - logger.info("The jdbc connection has created successfully!") - }) { e: Throwable => - logger.error(s"created data source connection error! $e") - return ErrorExecuteResponse("created data source connection error!", e) + /* url + user as the cache key */ + val jdbcUrl: String = properties.get(JDBCEngineConnConstant.JDBC_URL) + val execUser: String = properties.get(JDBCEngineConnConstant.JDBC_SCRIPTS_EXEC_USER) + val proxyUser: String = properties.get(JDBCEngineConnConstant.JDBC_PROXY_USER_PROPERTY) + var dataSourceIdentifier = s"$jdbcUrl-$execUser-$proxyUser" + /* If datasource is used, use datasource name as the cache key */ + if (StringUtils.isNotBlank(dataSourceName)) { + dataSourceIdentifier = s"$dataSourceName-$dataSourceMaxVersionId" + } + val connection = connectionManager.getConnection(dataSourceIdentifier, properties) + if (StringUtils.isNotBlank(taskId)) { + connectionCache.put(taskId, connection) } + connection + } + + override def executeLine( + engineExecutorContext: EngineExecutionContext, + code: String + ): ExecuteResponse = { + + val taskId = engineExecutorContext.getJobId.get + val connection: Connection = getConnection(engineExecutorContext) + var statement: Statement = null + var resultSet: ResultSet = null try { statement = connection.createStatement() @@ -167,14 +180,10 @@ class JDBCEngineConnExecutor(override val outputPrintLimit: Int, val id: Int) } } finally { if (resultSet != null) { - Utils.tryCatch({ resultSet.close() }) { case e: SQLException => - logger.warn(e.getMessage) - } + Utils.tryAndWarn(resultSet.close()) } if (statement != null) { - Utils.tryCatch({ statement.close() }) { case e: SQLException => - logger.warn(e.getMessage) - } + Utils.tryAndWarn(statement.close()) } } } catch { @@ -182,14 +191,6 @@ class JDBCEngineConnExecutor(override val outputPrintLimit: Int, val id: Int) logger.error(s"Cannot run $code", e) return ErrorExecuteResponse(e.getMessage, e) } finally { - if (connection != null) { - try { - if (!connection.getAutoCommit) connection.commit() - connection.close() - } catch { - case e: SQLException => logger.warn("close connection error.", e) - } - } connectionManager.removeStatement(taskId) } SuccessExecuteResponse() diff --git a/linkis-engineconn-plugins/nebula/src/main/java/org/apache/linkis/engineplugin/nebula/executor/NebulaEngineConnExecutor.java b/linkis-engineconn-plugins/nebula/src/main/java/org/apache/linkis/engineplugin/nebula/executor/NebulaEngineConnExecutor.java index 188ea60ec4..402afd6e17 100644 --- a/linkis-engineconn-plugins/nebula/src/main/java/org/apache/linkis/engineplugin/nebula/executor/NebulaEngineConnExecutor.java +++ b/linkis-engineconn-plugins/nebula/src/main/java/org/apache/linkis/engineplugin/nebula/executor/NebulaEngineConnExecutor.java @@ -333,7 +333,7 @@ private void queryOutput( .map( x -> { try { - return x.asString(); + return x.toString(); } catch (Exception e) { return ""; } diff --git a/linkis-engineconn-plugins/pom.xml b/linkis-engineconn-plugins/pom.xml index 5df6526748..054d300ed0 100644 --- a/linkis-engineconn-plugins/pom.xml +++ b/linkis-engineconn-plugins/pom.xml @@ -42,6 +42,8 @@ elasticsearch seatunnel hbase + nebula + doris repl diff --git a/linkis-engineconn-plugins/repl/src/main/java/org/apache/linkis/engineplugin/repl/executor/ReplEngineConnExecutor.java b/linkis-engineconn-plugins/repl/src/main/java/org/apache/linkis/engineplugin/repl/executor/ReplEngineConnExecutor.java index bf7201cb61..81f014e030 100644 --- a/linkis-engineconn-plugins/repl/src/main/java/org/apache/linkis/engineplugin/repl/executor/ReplEngineConnExecutor.java +++ b/linkis-engineconn-plugins/repl/src/main/java/org/apache/linkis/engineplugin/repl/executor/ReplEngineConnExecutor.java @@ -17,6 +17,9 @@ package org.apache.linkis.engineplugin.repl.executor; +import org.apache.linkis.common.io.MetaData; +import org.apache.linkis.common.io.Record; +import org.apache.linkis.common.io.resultset.ResultSetWriter; import org.apache.linkis.common.log.LogUtils; import org.apache.linkis.common.utils.OverloadUtils; import org.apache.linkis.engineconn.computation.executor.entity.EngineConnTask; @@ -36,15 +39,23 @@ import org.apache.linkis.manager.label.entity.engine.UserCreatorLabel; import org.apache.linkis.protocol.engine.JobProgressInfo; import org.apache.linkis.rpc.Sender; +import org.apache.linkis.scheduler.executer.ErrorExecuteResponse; import org.apache.linkis.scheduler.executer.ExecuteResponse; import org.apache.linkis.scheduler.executer.SuccessExecuteResponse; +import org.apache.linkis.storage.LineMetaData; +import org.apache.linkis.storage.LineRecord; +import org.apache.linkis.storage.resultset.ResultSetFactory; import org.apache.commons.collections4.MapUtils; +import org.apache.commons.io.IOUtils; import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.exception.ExceptionUtils; import org.springframework.util.CollectionUtils; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.PrintStream; import java.util.*; import java.util.concurrent.ConcurrentHashMap; @@ -132,14 +143,33 @@ public ExecuteResponse executeLine(EngineExecutionContext engineExecutorContext, threadCache.put(taskId, Thread.currentThread()); + ByteArrayOutputStream outputStream = new ByteArrayOutputStream(1024); + PrintStream cacheStream = new PrintStream(outputStream); + PrintStream oldStream = System.out; + System.setOut(cacheStream); + try { replAdapter.executorCode(realCode, classpathDir, methodName); } catch (Exception e) { String errorMessage = ExceptionUtils.getStackTrace(e); logger.error("Repl engine execute failed : {}", errorMessage); engineExecutorContext.appendStdout(LogUtils.generateERROR(errorMessage)); + return new ErrorExecuteResponse(errorMessage, null); } + String message = outputStream.toString(); + System.setOut(oldStream); + engineExecutorContext.appendStdout(message); + ResultSetWriter resultSetWriter = + engineExecutorContext.createResultSetWriter(ResultSetFactory.TEXT_TYPE); + try { + resultSetWriter.addMetaData(new LineMetaData()); + resultSetWriter.addRecord(new LineRecord(message)); + } catch (IOException e) { + logger.error("Failed to get the task result"); + } finally { + IOUtils.closeQuietly(resultSetWriter); + } return new SuccessExecuteResponse(); } diff --git a/linkis-engineconn-plugins/repl/src/main/java/org/apache/linkis/engineplugin/repl/executor/javarepl/JavaReplCompiler.java b/linkis-engineconn-plugins/repl/src/main/java/org/apache/linkis/engineplugin/repl/executor/javarepl/JavaReplCompiler.java index 7be4369735..9878ed5025 100644 --- a/linkis-engineconn-plugins/repl/src/main/java/org/apache/linkis/engineplugin/repl/executor/javarepl/JavaReplCompiler.java +++ b/linkis-engineconn-plugins/repl/src/main/java/org/apache/linkis/engineplugin/repl/executor/javarepl/JavaReplCompiler.java @@ -54,7 +54,7 @@ public class JavaReplCompiler { Pattern.compile("\\s+implements\\s+([\\w\\.]+)\\s*\\{\n"); private static final Pattern METHODS_PATTERN = - Pattern.compile("\n(private|public|protected)\\s+"); + Pattern.compile("(?<=\\})\\s+(private|public|protected)\\s+"); private static final Pattern FIELD_PATTERN = Pattern.compile("[^\n]+=[^\n]+;"); diff --git a/linkis-engineconn-plugins/repl/src/main/resources/log4j2.xml b/linkis-engineconn-plugins/repl/src/main/resources/log4j2.xml index 3e790b6dad..c75bf80fb3 100644 --- a/linkis-engineconn-plugins/repl/src/main/resources/log4j2.xml +++ b/linkis-engineconn-plugins/repl/src/main/resources/log4j2.xml @@ -47,7 +47,7 @@ - + diff --git a/linkis-engineconn-plugins/repl/src/main/resources/repl-ec.md b/linkis-engineconn-plugins/repl/src/main/resources/repl-ec.md index 5eb8c6c1b2..e04aec5d29 100644 --- a/linkis-engineconn-plugins/repl/src/main/resources/repl-ec.md +++ b/linkis-engineconn-plugins/repl/src/main/resources/repl-ec.md @@ -71,6 +71,43 @@ import org.apache.commons.lang3.StringUtils; } ``` +#### 1.3. Multiple methods + +```text +import org.apache.commons.lang3.StringUtils; + + public void sayHello() { + System.out.println("hello"); + System.out.println(StringUtils.isEmpty("hello")); + } + public void sayHi() { + System.out.println("hi"); + System.out.println(StringUtils.isEmpty("hi")); + } +``` + +```json +{ + "executionContent":{ + "code":"import org.apache.commons.lang3.StringUtils;\n\n public void sayHello() {\n System.out.println(\"hello\");\n System.out.println(StringUtils.isEmpty(\"hello\"));\n }\n public void sayHi() {\n System.out.println(\"hi\");\n System.out.println(StringUtils.isEmpty(\"hi\"));\n }", + "runType":"repl" + }, + "params":{ + "configuration":{ + "runtime":{ + "linkis.repl.type":"java", + "linkis.repl.method.name":"sayHi" + } + } + }, + "labels":{ + "engineType":"repl-1", + "userCreator":"linkis-IDE" + } +} +``` + + ### 2. Submitting scala tasks with Restful API ```text diff --git a/linkis-engineconn-plugins/spark/scala-2.12/pom.xml b/linkis-engineconn-plugins/spark/scala-2.12/pom.xml index 1ba1a8ad5f..840b02dc87 100644 --- a/linkis-engineconn-plugins/spark/scala-2.12/pom.xml +++ b/linkis-engineconn-plugins/spark/scala-2.12/pom.xml @@ -31,6 +31,10 @@ 2.0.2 0.13.0 8.11.0 + + + 3.2 + 1.2.0 @@ -185,6 +189,12 @@ + + org.apache.doris + spark-doris-connector-${spark.doris.version}_${scala.binary.version} + ${spark.doris.connector.version} + + com.lucidworks.spark spark-solr diff --git a/linkis-engineconn-plugins/spark/scala-2.12/src/test/scala/org/apache/linkis/engineplugin/spark/datacalc/TestDorisCala.scala b/linkis-engineconn-plugins/spark/scala-2.12/src/test/scala/org/apache/linkis/engineplugin/spark/datacalc/TestDorisCala.scala new file mode 100644 index 0000000000..4ddba1d2fd --- /dev/null +++ b/linkis-engineconn-plugins/spark/scala-2.12/src/test/scala/org/apache/linkis/engineplugin/spark/datacalc/TestDorisCala.scala @@ -0,0 +1,126 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.engineplugin.spark.datacalc + +import org.apache.linkis.common.io.FsPath +import org.apache.linkis.engineplugin.spark.datacalc.model.DataCalcGroupData + +import org.junit.jupiter.api.{Assertions, Test}; + +class TestDorisCala { + + val filePath = this.getClass.getResource("/").getFile + + @Test + def testDorisWrite: Unit = { + // skip os: windows + if (!FsPath.WINDOWS) { + val data = DataCalcGroupData.getData(dorisWriteConfigJson.replace("{filePath}", filePath)) + Assertions.assertTrue(data != null) + + val (sources, transforms, sinks) = DataCalcExecution.getPlugins(data) + Assertions.assertTrue(sources != null) + Assertions.assertTrue(transforms != null) + Assertions.assertTrue(sinks != null) + } + } + + @Test + def testDorisReader: Unit = { + // skip os: windows + if (!FsPath.WINDOWS) { + val data = + DataCalcGroupData.getData(dorisReaderConfigJson.replace("{filePath}", filePath)) + Assertions.assertTrue(data != null) + + val (sources, transforms, sinks) = DataCalcExecution.getPlugins(data) + Assertions.assertTrue(sources != null) + Assertions.assertTrue(transforms != null) + Assertions.assertTrue(sinks != null) + } + } + + val dorisWriteConfigJson = + """ + |{ + | "sources": [ + | { + | "name": "file", + | "type": "source", + | "config": { + | "resultTable": "T1654611700631", + | "path": "file://{filePath}/etltest.dolphin", + | "serializer": "csv", + | "options": { + | "header":"true", + | "delimiter":";" + | }, + | "columnNames": ["name", "age"] + | } + | } + | ], + | "sinks": [ + | { + | "name": "doris", + | "type": "sink", + | "config": { + | "sourceTable": "T1654611700631", + | "url": "localhost:8030", + | "user": "root", + | "password": "", + | "targetDatabase": "test", + | "targetTable": "test" + | } + | } + | ] + |} + |""".stripMargin + + val dorisReaderConfigJson = + """ + |{ + | "sources": [ + | { + | "name": "doris", + | "type": "source", + | "config": { + | "resultTable": "T1654611700631", + | "url": "localhost:8030", + | "user": "root", + | "password": "", + | "sourceDatabase": "test", + | "sourceTable": "test" + | } + | } + | ], + | "sinks": [ + | { + | "name": "file", + | "type": "sink", + | "config": { + | "sourceTable": "T1654611700631", + | "path": "file://{filePath}/json", + | "saveMode": "overwrite", + | "serializer": "json" + | } + | } + | ] + |} + |""".stripMargin + +} diff --git a/linkis-engineconn-plugins/spark/src/main/java/org/apache/linkis/engineplugin/spark/datacalc/sink/DorisSinkConfig.java b/linkis-engineconn-plugins/spark/src/main/java/org/apache/linkis/engineplugin/spark/datacalc/sink/DorisSinkConfig.java new file mode 100644 index 0000000000..3c8227faed --- /dev/null +++ b/linkis-engineconn-plugins/spark/src/main/java/org/apache/linkis/engineplugin/spark/datacalc/sink/DorisSinkConfig.java @@ -0,0 +1,91 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.engineplugin.spark.datacalc.sink; + +import org.apache.linkis.engineplugin.spark.datacalc.model.SinkConfig; + +import javax.validation.constraints.NotBlank; +import javax.validation.constraints.Pattern; + +public class DorisSinkConfig extends SinkConfig { + + @NotBlank private String url; + + @NotBlank private String user; + + private String password; + + @NotBlank private String targetDatabase; + + @NotBlank private String targetTable; + + @NotBlank + @Pattern( + regexp = "^(overwrite|append|ignore|error|errorifexists)$", + message = + "Unknown save mode: {saveMode}. Accepted save modes are 'overwrite', 'append', 'ignore', 'error', 'errorifexists'.") + private String saveMode = "overwrite"; + + public String getUrl() { + return url; + } + + public void setUrl(String url) { + this.url = url; + } + + public String getUser() { + return user; + } + + public void setUser(String user) { + this.user = user; + } + + public String getPassword() { + return password; + } + + public void setPassword(String password) { + this.password = password; + } + + public String getTargetDatabase() { + return targetDatabase; + } + + public void setTargetDatabase(String targetDatabase) { + this.targetDatabase = targetDatabase; + } + + public String getTargetTable() { + return targetTable; + } + + public void setTargetTable(String targetTable) { + this.targetTable = targetTable; + } + + public String getSaveMode() { + return saveMode; + } + + public void setSaveMode(String saveMode) { + this.saveMode = saveMode; + } +} diff --git a/linkis-engineconn-plugins/spark/src/main/java/org/apache/linkis/engineplugin/spark/datacalc/source/DorisSourceConfig.java b/linkis-engineconn-plugins/spark/src/main/java/org/apache/linkis/engineplugin/spark/datacalc/source/DorisSourceConfig.java new file mode 100644 index 0000000000..95a11d89df --- /dev/null +++ b/linkis-engineconn-plugins/spark/src/main/java/org/apache/linkis/engineplugin/spark/datacalc/source/DorisSourceConfig.java @@ -0,0 +1,74 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.engineplugin.spark.datacalc.source; + +import org.apache.linkis.engineplugin.spark.datacalc.model.SourceConfig; + +import javax.validation.constraints.NotBlank; + +public class DorisSourceConfig extends SourceConfig { + + @NotBlank private String url; + + @NotBlank private String user; + private String password; + + @NotBlank private String sourceDatabase; + + @NotBlank private String sourceTable; + + public String getUrl() { + return url; + } + + public void setUrl(String url) { + this.url = url; + } + + public String getUser() { + return user; + } + + public void setUser(String user) { + this.user = user; + } + + public String getPassword() { + return password; + } + + public void setPassword(String password) { + this.password = password; + } + + public String getSourceDatabase() { + return sourceDatabase; + } + + public void setSourceDatabase(String sourceDatabase) { + this.sourceDatabase = sourceDatabase; + } + + public String getSourceTable() { + return sourceTable; + } + + public void setSourceTable(String sourceTable) { + this.sourceTable = sourceTable; + } +} diff --git a/linkis-engineconn-plugins/spark/src/main/java/org/apache/linkis/engineplugin/spark/datacalc/util/PluginUtil.java b/linkis-engineconn-plugins/spark/src/main/java/org/apache/linkis/engineplugin/spark/datacalc/util/PluginUtil.java index e27d110c32..2d29c1b551 100644 --- a/linkis-engineconn-plugins/spark/src/main/java/org/apache/linkis/engineplugin/spark/datacalc/util/PluginUtil.java +++ b/linkis-engineconn-plugins/spark/src/main/java/org/apache/linkis/engineplugin/spark/datacalc/util/PluginUtil.java @@ -52,6 +52,7 @@ private static Map> getSourcePlugins() { classMap.put("solr", SolrSource.class); classMap.put("kafka", KafkaSource.class); classMap.put("starrocks", StarrocksSource.class); + classMap.put("doris", DorisSource.class); return classMap; } @@ -75,6 +76,7 @@ private static Map> getSinkPlugins() { classMap.put("solr", SolrSink.class); classMap.put("kafka", KafkaSink.class); classMap.put("starrocks", StarrocksSink.class); + classMap.put("doris", DorisSink.class); return classMap; } diff --git a/linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/datacalc/sink/DorisSink.scala b/linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/datacalc/sink/DorisSink.scala new file mode 100644 index 0000000000..9d5301ced9 --- /dev/null +++ b/linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/datacalc/sink/DorisSink.scala @@ -0,0 +1,57 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.engineplugin.spark.datacalc.sink + +import org.apache.linkis.common.utils.Logging +import org.apache.linkis.engineplugin.spark.datacalc.api.DataCalcSink + +import org.apache.commons.lang3.StringUtils +import org.apache.spark.sql.{Dataset, Row, SparkSession} + +import scala.collection.JavaConverters._ + +class DorisSink extends DataCalcSink[DorisSinkConfig] with Logging { + + def output(spark: SparkSession, ds: Dataset[Row]): Unit = { + var options = Map( + "doris.fenodes" -> config.getUrl, + "user" -> config.getUser, + "password" -> config.getPassword, + "doris.table.identifier" -> String.format( + "%s.%s", + config.getTargetDatabase, + config.getTargetTable + ) + ) + + if (config.getOptions != null && !config.getOptions.isEmpty) { + options = config.getOptions.asScala.toMap ++ options + } + + val writer = ds.write.format("doris") + if (StringUtils.isNotBlank(config.getSaveMode)) { + writer.mode(config.getSaveMode) + } + + logger.info( + s"Save data from doris url: ${config.getUrl}, targetDatabase: ${config.getTargetDatabase}, targetTable: ${config.getTargetTable}" + ) + writer.options(options).save() + } + +} diff --git a/linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/datacalc/source/DorisSource.scala b/linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/datacalc/source/DorisSource.scala new file mode 100644 index 0000000000..a4819f2181 --- /dev/null +++ b/linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/datacalc/source/DorisSource.scala @@ -0,0 +1,49 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.engineplugin.spark.datacalc.source + +import org.apache.linkis.common.utils.Logging +import org.apache.linkis.engineplugin.spark.datacalc.api.DataCalcSource + +import org.apache.spark.sql.{Dataset, Row, SparkSession} + +class DorisSource extends DataCalcSource[DorisSourceConfig] with Logging { + + override def getData(spark: SparkSession): Dataset[Row] = { + val reader = spark.read.format("doris") + + if (config.getOptions != null && !config.getOptions.isEmpty) { + reader.options(config.getOptions) + } + + logger.info( + s"Load data from Doris url: ${config.getUrl}, sourceDatabase: ${config.getSourceDatabase}, sourceTable: ${config.getSourceTable}" + ) + + reader + .option( + "doris.table.identifier", + String.format("%s.%s", config.getSourceDatabase, config.getSourceTable) + ) + .option("doris.fenodes", config.getUrl) + .option("user", config.getUser) + .option("password", config.getPassword) + .load() + } + +} diff --git a/linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/executor/SQLSession.scala b/linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/executor/SQLSession.scala index 77fe9ca973..6e302b4402 100644 --- a/linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/executor/SQLSession.scala +++ b/linkis-engineconn-plugins/spark/src/main/scala/org/apache/linkis/engineplugin/spark/executor/SQLSession.scala @@ -69,6 +69,7 @@ object SQLSession extends Logging { "Spark application sc has already stopped, please restart it." ) } + val startTime = System.currentTimeMillis() // sc.setJobGroup(jobGroup, "Get IDE-SQL Results.", false) diff --git a/linkis-engineconn-plugins/sqoop/pom.xml b/linkis-engineconn-plugins/sqoop/pom.xml index 180f12d967..76319e98bf 100644 --- a/linkis-engineconn-plugins/sqoop/pom.xml +++ b/linkis-engineconn-plugins/sqoop/pom.xml @@ -41,7 +41,7 @@ org.apache.avro avro - 1.11.0 + 1.11.3 provided diff --git a/linkis-public-enhancements/distribution.xml b/linkis-public-enhancements/distribution.xml index 0ac5bf5f8c..b000fc8539 100644 --- a/linkis-public-enhancements/distribution.xml +++ b/linkis-public-enhancements/distribution.xml @@ -290,10 +290,10 @@ - ./linkis-datasource/linkis-metadata-query/server/target/out/lib/service + ./linkis-datasource/linkis-datasource-manager/server/target/out/lib/service - lib/metadataquery-service + lib/metadataquery-service **/* diff --git a/linkis-public-enhancements/linkis-configuration/src/main/java/org/apache/linkis/configuration/restful/api/TemplateManagerRestfulApi.java b/linkis-public-enhancements/linkis-configuration/src/main/java/org/apache/linkis/configuration/restful/api/TemplateManagerRestfulApi.java index 7ed5c4e579..e0d52d423d 100644 --- a/linkis-public-enhancements/linkis-configuration/src/main/java/org/apache/linkis/configuration/restful/api/TemplateManagerRestfulApi.java +++ b/linkis-public-enhancements/linkis-configuration/src/main/java/org/apache/linkis/configuration/restful/api/TemplateManagerRestfulApi.java @@ -86,8 +86,7 @@ public Message updateKeyMapping(HttpServletRequest req, @RequestBody JsonNode js // check special admin token if (StringUtils.isNotBlank(token)) { if (!Configuration.isAdminToken(token)) { - logger.warn("Token:{} has no permission to updateKeyMapping.", token); - return Message.error("Token:" + token + " has no permission to updateKeyMapping."); + return Message.error("Token has no permission to updateKeyMapping."); } } else if (!Configuration.isAdmin(username)) { logger.warn("User:{} has no permission to updateKeyMapping.", username); @@ -160,8 +159,7 @@ public Message queryKeyInfoList(HttpServletRequest req, @RequestBody JsonNode js // check special admin token if (StringUtils.isNotBlank(token)) { if (!Configuration.isAdminToken(token)) { - logger.warn("Token:{} has no permission to queryKeyInfoList.", token); - return Message.error("Token:" + token + " has no permission to queryKeyInfoList."); + return Message.error("Token has no permission to queryKeyInfoList."); } } else if (!Configuration.isAdmin(username)) { logger.warn("User:{} has no permission to queryKeyInfoList.", username); @@ -212,8 +210,7 @@ public Message apply(HttpServletRequest req, @RequestBody JsonNode jsonNode) // check special admin token if (StringUtils.isNotBlank(token)) { if (!Configuration.isAdminToken(token)) { - logger.warn("Token:{} has no permission to apply.", token); - return Message.error("Token:" + token + " has no permission to apply."); + return Message.error("Token has no permission to apply."); } } else if (!Configuration.isAdmin(username)) { logger.warn("User:{} has no permission to apply.", username); diff --git a/linkis-public-enhancements/linkis-datasource/linkis-datasource-manager/server/src/main/java/org/apache/linkis/datasourcemanager/core/restful/RestfulApiHelper.java b/linkis-public-enhancements/linkis-datasource/linkis-datasource-manager/server/src/main/java/org/apache/linkis/datasourcemanager/core/restful/RestfulApiHelper.java index 9651bb265b..72f5c9f899 100644 --- a/linkis-public-enhancements/linkis-datasource/linkis-datasource-manager/server/src/main/java/org/apache/linkis/datasourcemanager/core/restful/RestfulApiHelper.java +++ b/linkis-public-enhancements/linkis-datasource/linkis-datasource-manager/server/src/main/java/org/apache/linkis/datasourcemanager/core/restful/RestfulApiHelper.java @@ -66,10 +66,11 @@ public static void encryptPasswordKey( keyDefinitionList.forEach( keyDefinition -> { if (keyDefinition.getValueType() == DataSourceParamKeyDefinition.ValueType.PASSWORD) { - String password = String.valueOf(connectParams.get(keyDefinition.getKey())); + Object password = connectParams.get(keyDefinition.getKey()); if (null != password) { connectParams.put( - keyDefinition.getKey(), new String(new Base64().encode(password.getBytes()))); + keyDefinition.getKey(), + new String(new Base64().encode(String.valueOf(password).getBytes()))); } } }); @@ -86,10 +87,11 @@ public static void decryptPasswordKey( keyDefinitionList.forEach( keyDefinition -> { if (keyDefinition.getValueType() == DataSourceParamKeyDefinition.ValueType.PASSWORD) { - String password = String.valueOf(connectParams.get(keyDefinition.getKey())); + Object password = connectParams.get(keyDefinition.getKey()); if (null != password) { connectParams.put( - keyDefinition.getKey(), new String(new Base64().decode(password.getBytes()))); + keyDefinition.getKey(), + new String(new Base64().decode(String.valueOf(password).getBytes()))); } } }); diff --git a/linkis-public-enhancements/linkis-datasource/linkis-metadata-query/service/elasticsearch/pom.xml b/linkis-public-enhancements/linkis-datasource/linkis-datasource-manager/service/elasticsearch/pom.xml similarity index 100% rename from linkis-public-enhancements/linkis-datasource/linkis-metadata-query/service/elasticsearch/pom.xml rename to linkis-public-enhancements/linkis-datasource/linkis-datasource-manager/service/elasticsearch/pom.xml diff --git a/linkis-public-enhancements/linkis-datasource/linkis-metadata-query/service/elasticsearch/src/main/assembly/distribution.xml b/linkis-public-enhancements/linkis-datasource/linkis-datasource-manager/service/elasticsearch/src/main/assembly/distribution.xml similarity index 100% rename from linkis-public-enhancements/linkis-datasource/linkis-metadata-query/service/elasticsearch/src/main/assembly/distribution.xml rename to linkis-public-enhancements/linkis-datasource/linkis-datasource-manager/service/elasticsearch/src/main/assembly/distribution.xml diff --git a/linkis-public-enhancements/linkis-datasource/linkis-metadata-query/service/elasticsearch/src/main/java/org/apache/linkis/metadata/query/service/ElasticConnection.java b/linkis-public-enhancements/linkis-datasource/linkis-datasource-manager/service/elasticsearch/src/main/java/org/apache/linkis/metadata/query/service/ElasticConnection.java similarity index 96% rename from linkis-public-enhancements/linkis-datasource/linkis-metadata-query/service/elasticsearch/src/main/java/org/apache/linkis/metadata/query/service/ElasticConnection.java rename to linkis-public-enhancements/linkis-datasource/linkis-datasource-manager/service/elasticsearch/src/main/java/org/apache/linkis/metadata/query/service/ElasticConnection.java index 5f47452cd6..24aea200e5 100644 --- a/linkis-public-enhancements/linkis-datasource/linkis-metadata-query/service/elasticsearch/src/main/java/org/apache/linkis/metadata/query/service/ElasticConnection.java +++ b/linkis-public-enhancements/linkis-datasource/linkis-datasource-manager/service/elasticsearch/src/main/java/org/apache/linkis/metadata/query/service/ElasticConnection.java @@ -111,11 +111,11 @@ public List getTypes(String index) throws Exception { } public Map getProps(String index, String type) throws Exception { - Request request = new Request("GET", index + "/_mappings/" + type); + Request request = new Request("GET", index + "/_mappings"); Response response = restClient.performRequest(request); Map> result = Json.fromJson(response.getEntity().getContent(), Map.class); - Map mappings = (Map) result.get(index).get("mappings"); + Map mappings = (Map) result.get(index).get(DEFAULT_MAPPING_NAME); Map propsMap = mappings; if (mappings.containsKey(type)) { Object typeMap = mappings.get(type); @@ -124,10 +124,10 @@ public Map getProps(String index, String type) throws Exception } } Object props = propsMap.get(FIELD_PROPS); - if (props instanceof Map) { + if (null != props && props instanceof Map) { return (Map) props; } - return null; + return propsMap; } public void ping() throws IOException { diff --git a/linkis-public-enhancements/linkis-datasource/linkis-metadata-query/service/elasticsearch/src/main/java/org/apache/linkis/metadata/query/service/ElasticParamsMapper.java b/linkis-public-enhancements/linkis-datasource/linkis-datasource-manager/service/elasticsearch/src/main/java/org/apache/linkis/metadata/query/service/ElasticParamsMapper.java similarity index 100% rename from linkis-public-enhancements/linkis-datasource/linkis-metadata-query/service/elasticsearch/src/main/java/org/apache/linkis/metadata/query/service/ElasticParamsMapper.java rename to linkis-public-enhancements/linkis-datasource/linkis-datasource-manager/service/elasticsearch/src/main/java/org/apache/linkis/metadata/query/service/ElasticParamsMapper.java diff --git a/linkis-public-enhancements/linkis-datasource/linkis-metadata-query/service/elasticsearch/src/main/java/org/apache/linkis/metadata/query/service/EsMetaService.java b/linkis-public-enhancements/linkis-datasource/linkis-datasource-manager/service/elasticsearch/src/main/java/org/apache/linkis/metadata/query/service/EsMetaService.java similarity index 100% rename from linkis-public-enhancements/linkis-datasource/linkis-metadata-query/service/elasticsearch/src/main/java/org/apache/linkis/metadata/query/service/EsMetaService.java rename to linkis-public-enhancements/linkis-datasource/linkis-datasource-manager/service/elasticsearch/src/main/java/org/apache/linkis/metadata/query/service/EsMetaService.java diff --git a/linkis-public-enhancements/linkis-datasource/linkis-metadata-query/service/hdfs/pom.xml b/linkis-public-enhancements/linkis-datasource/linkis-datasource-manager/service/hdfs/pom.xml similarity index 100% rename from linkis-public-enhancements/linkis-datasource/linkis-metadata-query/service/hdfs/pom.xml rename to linkis-public-enhancements/linkis-datasource/linkis-datasource-manager/service/hdfs/pom.xml diff --git a/linkis-public-enhancements/linkis-datasource/linkis-metadata-query/service/hdfs/src/main/assembly/distribution.xml b/linkis-public-enhancements/linkis-datasource/linkis-datasource-manager/service/hdfs/src/main/assembly/distribution.xml similarity index 100% rename from linkis-public-enhancements/linkis-datasource/linkis-metadata-query/service/hdfs/src/main/assembly/distribution.xml rename to linkis-public-enhancements/linkis-datasource/linkis-datasource-manager/service/hdfs/src/main/assembly/distribution.xml diff --git a/linkis-public-enhancements/linkis-datasource/linkis-metadata-query/service/hdfs/src/main/java/org/apache/linkis/metadata/query/service/HdfsConnection.java b/linkis-public-enhancements/linkis-datasource/linkis-datasource-manager/service/hdfs/src/main/java/org/apache/linkis/metadata/query/service/HdfsConnection.java similarity index 100% rename from linkis-public-enhancements/linkis-datasource/linkis-metadata-query/service/hdfs/src/main/java/org/apache/linkis/metadata/query/service/HdfsConnection.java rename to linkis-public-enhancements/linkis-datasource/linkis-datasource-manager/service/hdfs/src/main/java/org/apache/linkis/metadata/query/service/HdfsConnection.java diff --git a/linkis-public-enhancements/linkis-datasource/linkis-metadata-query/service/hdfs/src/main/java/org/apache/linkis/metadata/query/service/HdfsMetaService.java b/linkis-public-enhancements/linkis-datasource/linkis-datasource-manager/service/hdfs/src/main/java/org/apache/linkis/metadata/query/service/HdfsMetaService.java similarity index 100% rename from linkis-public-enhancements/linkis-datasource/linkis-metadata-query/service/hdfs/src/main/java/org/apache/linkis/metadata/query/service/HdfsMetaService.java rename to linkis-public-enhancements/linkis-datasource/linkis-datasource-manager/service/hdfs/src/main/java/org/apache/linkis/metadata/query/service/HdfsMetaService.java diff --git a/linkis-public-enhancements/linkis-datasource/linkis-metadata-query/service/hdfs/src/main/java/org/apache/linkis/metadata/query/service/HdfsParamsMapper.java b/linkis-public-enhancements/linkis-datasource/linkis-datasource-manager/service/hdfs/src/main/java/org/apache/linkis/metadata/query/service/HdfsParamsMapper.java similarity index 100% rename from linkis-public-enhancements/linkis-datasource/linkis-metadata-query/service/hdfs/src/main/java/org/apache/linkis/metadata/query/service/HdfsParamsMapper.java rename to linkis-public-enhancements/linkis-datasource/linkis-datasource-manager/service/hdfs/src/main/java/org/apache/linkis/metadata/query/service/HdfsParamsMapper.java diff --git a/linkis-public-enhancements/linkis-datasource/linkis-metadata-query/service/hdfs/src/main/java/org/apache/linkis/metadata/query/service/conf/ConfigurationUtils.java b/linkis-public-enhancements/linkis-datasource/linkis-datasource-manager/service/hdfs/src/main/java/org/apache/linkis/metadata/query/service/conf/ConfigurationUtils.java similarity index 100% rename from linkis-public-enhancements/linkis-datasource/linkis-metadata-query/service/hdfs/src/main/java/org/apache/linkis/metadata/query/service/conf/ConfigurationUtils.java rename to linkis-public-enhancements/linkis-datasource/linkis-datasource-manager/service/hdfs/src/main/java/org/apache/linkis/metadata/query/service/conf/ConfigurationUtils.java diff --git a/linkis-public-enhancements/linkis-datasource/linkis-metadata-query/service/hive/pom.xml b/linkis-public-enhancements/linkis-datasource/linkis-datasource-manager/service/hive/pom.xml similarity index 100% rename from linkis-public-enhancements/linkis-datasource/linkis-metadata-query/service/hive/pom.xml rename to linkis-public-enhancements/linkis-datasource/linkis-datasource-manager/service/hive/pom.xml diff --git a/linkis-public-enhancements/linkis-datasource/linkis-metadata-query/service/hive/src/main/assembly/distribution.xml b/linkis-public-enhancements/linkis-datasource/linkis-datasource-manager/service/hive/src/main/assembly/distribution.xml similarity index 100% rename from linkis-public-enhancements/linkis-datasource/linkis-metadata-query/service/hive/src/main/assembly/distribution.xml rename to linkis-public-enhancements/linkis-datasource/linkis-datasource-manager/service/hive/src/main/assembly/distribution.xml diff --git a/linkis-public-enhancements/linkis-datasource/linkis-metadata-query/service/hive/src/main/java/org/apache/linkis/metadata/query/service/HiveConnection.java b/linkis-public-enhancements/linkis-datasource/linkis-datasource-manager/service/hive/src/main/java/org/apache/linkis/metadata/query/service/HiveConnection.java similarity index 100% rename from linkis-public-enhancements/linkis-datasource/linkis-metadata-query/service/hive/src/main/java/org/apache/linkis/metadata/query/service/HiveConnection.java rename to linkis-public-enhancements/linkis-datasource/linkis-datasource-manager/service/hive/src/main/java/org/apache/linkis/metadata/query/service/HiveConnection.java diff --git a/linkis-public-enhancements/linkis-datasource/linkis-metadata-query/service/hive/src/main/java/org/apache/linkis/metadata/query/service/HiveMetaService.java b/linkis-public-enhancements/linkis-datasource/linkis-datasource-manager/service/hive/src/main/java/org/apache/linkis/metadata/query/service/HiveMetaService.java similarity index 100% rename from linkis-public-enhancements/linkis-datasource/linkis-metadata-query/service/hive/src/main/java/org/apache/linkis/metadata/query/service/HiveMetaService.java rename to linkis-public-enhancements/linkis-datasource/linkis-datasource-manager/service/hive/src/main/java/org/apache/linkis/metadata/query/service/HiveMetaService.java diff --git a/linkis-public-enhancements/linkis-datasource/linkis-metadata-query/service/hive/src/main/java/org/apache/linkis/metadata/query/service/HiveParamsMapper.java b/linkis-public-enhancements/linkis-datasource/linkis-datasource-manager/service/hive/src/main/java/org/apache/linkis/metadata/query/service/HiveParamsMapper.java similarity index 100% rename from linkis-public-enhancements/linkis-datasource/linkis-metadata-query/service/hive/src/main/java/org/apache/linkis/metadata/query/service/HiveParamsMapper.java rename to linkis-public-enhancements/linkis-datasource/linkis-datasource-manager/service/hive/src/main/java/org/apache/linkis/metadata/query/service/HiveParamsMapper.java diff --git a/linkis-public-enhancements/linkis-datasource/linkis-metadata-query/service/jdbc/pom.xml b/linkis-public-enhancements/linkis-datasource/linkis-datasource-manager/service/jdbc/pom.xml similarity index 100% rename from linkis-public-enhancements/linkis-datasource/linkis-metadata-query/service/jdbc/pom.xml rename to linkis-public-enhancements/linkis-datasource/linkis-datasource-manager/service/jdbc/pom.xml diff --git a/linkis-public-enhancements/linkis-datasource/linkis-metadata-query/service/jdbc/src/main/assembly/distribution.xml b/linkis-public-enhancements/linkis-datasource/linkis-datasource-manager/service/jdbc/src/main/assembly/distribution.xml similarity index 100% rename from linkis-public-enhancements/linkis-datasource/linkis-metadata-query/service/jdbc/src/main/assembly/distribution.xml rename to linkis-public-enhancements/linkis-datasource/linkis-datasource-manager/service/jdbc/src/main/assembly/distribution.xml diff --git a/linkis-public-enhancements/linkis-datasource/linkis-metadata-query/service/jdbc/src/main/java/org/apache/linkis/metadata/query/service/AbstractSqlConnection.java b/linkis-public-enhancements/linkis-datasource/linkis-datasource-manager/service/jdbc/src/main/java/org/apache/linkis/metadata/query/service/AbstractSqlConnection.java similarity index 100% rename from linkis-public-enhancements/linkis-datasource/linkis-metadata-query/service/jdbc/src/main/java/org/apache/linkis/metadata/query/service/AbstractSqlConnection.java rename to linkis-public-enhancements/linkis-datasource/linkis-datasource-manager/service/jdbc/src/main/java/org/apache/linkis/metadata/query/service/AbstractSqlConnection.java diff --git a/linkis-public-enhancements/linkis-datasource/linkis-metadata-query/service/jdbc/src/main/java/org/apache/linkis/metadata/query/service/ClickhouseMetaService.java b/linkis-public-enhancements/linkis-datasource/linkis-datasource-manager/service/jdbc/src/main/java/org/apache/linkis/metadata/query/service/ClickhouseMetaService.java similarity index 100% rename from linkis-public-enhancements/linkis-datasource/linkis-metadata-query/service/jdbc/src/main/java/org/apache/linkis/metadata/query/service/ClickhouseMetaService.java rename to linkis-public-enhancements/linkis-datasource/linkis-datasource-manager/service/jdbc/src/main/java/org/apache/linkis/metadata/query/service/ClickhouseMetaService.java diff --git a/linkis-public-enhancements/linkis-datasource/linkis-metadata-query/service/jdbc/src/main/java/org/apache/linkis/metadata/query/service/Db2MetaService.java b/linkis-public-enhancements/linkis-datasource/linkis-datasource-manager/service/jdbc/src/main/java/org/apache/linkis/metadata/query/service/Db2MetaService.java similarity index 100% rename from linkis-public-enhancements/linkis-datasource/linkis-metadata-query/service/jdbc/src/main/java/org/apache/linkis/metadata/query/service/Db2MetaService.java rename to linkis-public-enhancements/linkis-datasource/linkis-datasource-manager/service/jdbc/src/main/java/org/apache/linkis/metadata/query/service/Db2MetaService.java diff --git a/linkis-public-enhancements/linkis-datasource/linkis-metadata-query/service/jdbc/src/main/java/org/apache/linkis/metadata/query/service/DmMetaService.java b/linkis-public-enhancements/linkis-datasource/linkis-datasource-manager/service/jdbc/src/main/java/org/apache/linkis/metadata/query/service/DmMetaService.java similarity index 100% rename from linkis-public-enhancements/linkis-datasource/linkis-metadata-query/service/jdbc/src/main/java/org/apache/linkis/metadata/query/service/DmMetaService.java rename to linkis-public-enhancements/linkis-datasource/linkis-datasource-manager/service/jdbc/src/main/java/org/apache/linkis/metadata/query/service/DmMetaService.java diff --git a/linkis-public-enhancements/linkis-datasource/linkis-metadata-query/service/jdbc/src/main/java/org/apache/linkis/metadata/query/service/GreenplumMetaService.java b/linkis-public-enhancements/linkis-datasource/linkis-datasource-manager/service/jdbc/src/main/java/org/apache/linkis/metadata/query/service/GreenplumMetaService.java similarity index 100% rename from linkis-public-enhancements/linkis-datasource/linkis-metadata-query/service/jdbc/src/main/java/org/apache/linkis/metadata/query/service/GreenplumMetaService.java rename to linkis-public-enhancements/linkis-datasource/linkis-datasource-manager/service/jdbc/src/main/java/org/apache/linkis/metadata/query/service/GreenplumMetaService.java diff --git a/linkis-public-enhancements/linkis-datasource/linkis-metadata-query/service/jdbc/src/main/java/org/apache/linkis/metadata/query/service/KingbaseMetaService.java b/linkis-public-enhancements/linkis-datasource/linkis-datasource-manager/service/jdbc/src/main/java/org/apache/linkis/metadata/query/service/KingbaseMetaService.java similarity index 100% rename from linkis-public-enhancements/linkis-datasource/linkis-metadata-query/service/jdbc/src/main/java/org/apache/linkis/metadata/query/service/KingbaseMetaService.java rename to linkis-public-enhancements/linkis-datasource/linkis-datasource-manager/service/jdbc/src/main/java/org/apache/linkis/metadata/query/service/KingbaseMetaService.java diff --git a/linkis-public-enhancements/linkis-datasource/linkis-metadata-query/service/jdbc/src/main/java/org/apache/linkis/metadata/query/service/MysqlMetaService.java b/linkis-public-enhancements/linkis-datasource/linkis-datasource-manager/service/jdbc/src/main/java/org/apache/linkis/metadata/query/service/MysqlMetaService.java similarity index 100% rename from linkis-public-enhancements/linkis-datasource/linkis-metadata-query/service/jdbc/src/main/java/org/apache/linkis/metadata/query/service/MysqlMetaService.java rename to linkis-public-enhancements/linkis-datasource/linkis-datasource-manager/service/jdbc/src/main/java/org/apache/linkis/metadata/query/service/MysqlMetaService.java diff --git a/linkis-public-enhancements/linkis-datasource/linkis-metadata-query/service/jdbc/src/main/java/org/apache/linkis/metadata/query/service/OracleMetaService.java b/linkis-public-enhancements/linkis-datasource/linkis-datasource-manager/service/jdbc/src/main/java/org/apache/linkis/metadata/query/service/OracleMetaService.java similarity index 98% rename from linkis-public-enhancements/linkis-datasource/linkis-metadata-query/service/jdbc/src/main/java/org/apache/linkis/metadata/query/service/OracleMetaService.java rename to linkis-public-enhancements/linkis-datasource/linkis-datasource-manager/service/jdbc/src/main/java/org/apache/linkis/metadata/query/service/OracleMetaService.java index 71f6595952..e048d71bda 100644 --- a/linkis-public-enhancements/linkis-datasource/linkis-metadata-query/service/jdbc/src/main/java/org/apache/linkis/metadata/query/service/OracleMetaService.java +++ b/linkis-public-enhancements/linkis-datasource/linkis-datasource-manager/service/jdbc/src/main/java/org/apache/linkis/metadata/query/service/OracleMetaService.java @@ -74,11 +74,10 @@ public MetadataConnection getConnection( assert extraParams != null; LOG.info("oracle connection params:{}", params.toString()); LOG.info( - "oracle connection host:{},port:{},username:{},password:{},database:{}", + "oracle connection host:{},port:{},username:{},database:{}", host, port, username, - password, database); return new MetadataConnection<>( new SqlConnection(host, port, username, password, database, serviceName, extraParams)); diff --git a/linkis-public-enhancements/linkis-datasource/linkis-metadata-query/service/jdbc/src/main/java/org/apache/linkis/metadata/query/service/PostgresqlMetaService.java b/linkis-public-enhancements/linkis-datasource/linkis-datasource-manager/service/jdbc/src/main/java/org/apache/linkis/metadata/query/service/PostgresqlMetaService.java similarity index 100% rename from linkis-public-enhancements/linkis-datasource/linkis-metadata-query/service/jdbc/src/main/java/org/apache/linkis/metadata/query/service/PostgresqlMetaService.java rename to linkis-public-enhancements/linkis-datasource/linkis-datasource-manager/service/jdbc/src/main/java/org/apache/linkis/metadata/query/service/PostgresqlMetaService.java diff --git a/linkis-public-enhancements/linkis-datasource/linkis-metadata-query/service/jdbc/src/main/java/org/apache/linkis/metadata/query/service/SqlserverMetaService.java b/linkis-public-enhancements/linkis-datasource/linkis-datasource-manager/service/jdbc/src/main/java/org/apache/linkis/metadata/query/service/SqlserverMetaService.java similarity index 100% rename from linkis-public-enhancements/linkis-datasource/linkis-metadata-query/service/jdbc/src/main/java/org/apache/linkis/metadata/query/service/SqlserverMetaService.java rename to linkis-public-enhancements/linkis-datasource/linkis-datasource-manager/service/jdbc/src/main/java/org/apache/linkis/metadata/query/service/SqlserverMetaService.java diff --git a/linkis-public-enhancements/linkis-datasource/linkis-metadata-query/service/jdbc/src/main/java/org/apache/linkis/metadata/query/service/clickhouse/SqlConnection.java b/linkis-public-enhancements/linkis-datasource/linkis-datasource-manager/service/jdbc/src/main/java/org/apache/linkis/metadata/query/service/clickhouse/SqlConnection.java similarity index 100% rename from linkis-public-enhancements/linkis-datasource/linkis-metadata-query/service/jdbc/src/main/java/org/apache/linkis/metadata/query/service/clickhouse/SqlConnection.java rename to linkis-public-enhancements/linkis-datasource/linkis-datasource-manager/service/jdbc/src/main/java/org/apache/linkis/metadata/query/service/clickhouse/SqlConnection.java diff --git a/linkis-public-enhancements/linkis-datasource/linkis-metadata-query/service/jdbc/src/main/java/org/apache/linkis/metadata/query/service/conf/SqlParamsMapper.java b/linkis-public-enhancements/linkis-datasource/linkis-datasource-manager/service/jdbc/src/main/java/org/apache/linkis/metadata/query/service/conf/SqlParamsMapper.java similarity index 100% rename from linkis-public-enhancements/linkis-datasource/linkis-metadata-query/service/jdbc/src/main/java/org/apache/linkis/metadata/query/service/conf/SqlParamsMapper.java rename to linkis-public-enhancements/linkis-datasource/linkis-datasource-manager/service/jdbc/src/main/java/org/apache/linkis/metadata/query/service/conf/SqlParamsMapper.java diff --git a/linkis-public-enhancements/linkis-datasource/linkis-metadata-query/service/jdbc/src/main/java/org/apache/linkis/metadata/query/service/db2/SqlConnection.java b/linkis-public-enhancements/linkis-datasource/linkis-datasource-manager/service/jdbc/src/main/java/org/apache/linkis/metadata/query/service/db2/SqlConnection.java similarity index 90% rename from linkis-public-enhancements/linkis-datasource/linkis-metadata-query/service/jdbc/src/main/java/org/apache/linkis/metadata/query/service/db2/SqlConnection.java rename to linkis-public-enhancements/linkis-datasource/linkis-datasource-manager/service/jdbc/src/main/java/org/apache/linkis/metadata/query/service/db2/SqlConnection.java index 3f61ac51f5..09201d58d8 100644 --- a/linkis-public-enhancements/linkis-datasource/linkis-metadata-query/service/jdbc/src/main/java/org/apache/linkis/metadata/query/service/db2/SqlConnection.java +++ b/linkis-public-enhancements/linkis-datasource/linkis-datasource-manager/service/jdbc/src/main/java/org/apache/linkis/metadata/query/service/db2/SqlConnection.java @@ -18,6 +18,7 @@ package org.apache.linkis.metadata.query.service.db2; import org.apache.linkis.common.conf.CommonVars; +import org.apache.linkis.common.exception.LinkisSecurityException; import org.apache.linkis.metadata.query.service.AbstractSqlConnection; import org.apache.commons.collections.MapUtils; @@ -42,6 +43,10 @@ public class SqlConnection extends AbstractSqlConnection { private static final CommonVars SQL_CONNECT_URL = CommonVars.apply("wds.linkis.server.mdm.service.db2.url", "jdbc:db2://%s:%s/%s"); + /** clientRerouteServerListJNDIName */ + private static final CommonVars DB2_SENSITIVE_PARAMS = + CommonVars.apply("linkis.db2.sensitive.params", "clientRerouteServerListJNDIName"); + public SqlConnection( String host, Integer port, @@ -115,6 +120,9 @@ public Connection getDBConnection(ConnectMessage connectMessage, String database .collect(Collectors.joining("&")); url += "?" + extraParamString; } + if (url.toLowerCase().contains(DB2_SENSITIVE_PARAMS.getValue().toLowerCase())) { + throw new LinkisSecurityException(35000, "Invalid db2 connection params."); + } return DriverManager.getConnection(url, connectMessage.username, connectMessage.password); } diff --git a/linkis-public-enhancements/linkis-datasource/linkis-metadata-query/service/jdbc/src/main/java/org/apache/linkis/metadata/query/service/dm/SqlConnection.java b/linkis-public-enhancements/linkis-datasource/linkis-datasource-manager/service/jdbc/src/main/java/org/apache/linkis/metadata/query/service/dm/SqlConnection.java similarity index 100% rename from linkis-public-enhancements/linkis-datasource/linkis-metadata-query/service/jdbc/src/main/java/org/apache/linkis/metadata/query/service/dm/SqlConnection.java rename to linkis-public-enhancements/linkis-datasource/linkis-datasource-manager/service/jdbc/src/main/java/org/apache/linkis/metadata/query/service/dm/SqlConnection.java diff --git a/linkis-public-enhancements/linkis-datasource/linkis-metadata-query/service/jdbc/src/main/java/org/apache/linkis/metadata/query/service/greenplum/SqlConnection.java b/linkis-public-enhancements/linkis-datasource/linkis-datasource-manager/service/jdbc/src/main/java/org/apache/linkis/metadata/query/service/greenplum/SqlConnection.java similarity index 100% rename from linkis-public-enhancements/linkis-datasource/linkis-metadata-query/service/jdbc/src/main/java/org/apache/linkis/metadata/query/service/greenplum/SqlConnection.java rename to linkis-public-enhancements/linkis-datasource/linkis-datasource-manager/service/jdbc/src/main/java/org/apache/linkis/metadata/query/service/greenplum/SqlConnection.java diff --git a/linkis-public-enhancements/linkis-datasource/linkis-metadata-query/service/jdbc/src/main/java/org/apache/linkis/metadata/query/service/kingbase/SqlConnection.java b/linkis-public-enhancements/linkis-datasource/linkis-datasource-manager/service/jdbc/src/main/java/org/apache/linkis/metadata/query/service/kingbase/SqlConnection.java similarity index 100% rename from linkis-public-enhancements/linkis-datasource/linkis-metadata-query/service/jdbc/src/main/java/org/apache/linkis/metadata/query/service/kingbase/SqlConnection.java rename to linkis-public-enhancements/linkis-datasource/linkis-datasource-manager/service/jdbc/src/main/java/org/apache/linkis/metadata/query/service/kingbase/SqlConnection.java diff --git a/linkis-public-enhancements/linkis-datasource/linkis-metadata-query/service/jdbc/src/main/java/org/apache/linkis/metadata/query/service/mysql/SqlConnection.java b/linkis-public-enhancements/linkis-datasource/linkis-datasource-manager/service/jdbc/src/main/java/org/apache/linkis/metadata/query/service/mysql/SqlConnection.java similarity index 100% rename from linkis-public-enhancements/linkis-datasource/linkis-metadata-query/service/jdbc/src/main/java/org/apache/linkis/metadata/query/service/mysql/SqlConnection.java rename to linkis-public-enhancements/linkis-datasource/linkis-datasource-manager/service/jdbc/src/main/java/org/apache/linkis/metadata/query/service/mysql/SqlConnection.java diff --git a/linkis-public-enhancements/linkis-datasource/linkis-metadata-query/service/jdbc/src/main/java/org/apache/linkis/metadata/query/service/oracle/SqlConnection.java b/linkis-public-enhancements/linkis-datasource/linkis-datasource-manager/service/jdbc/src/main/java/org/apache/linkis/metadata/query/service/oracle/SqlConnection.java similarity index 100% rename from linkis-public-enhancements/linkis-datasource/linkis-metadata-query/service/jdbc/src/main/java/org/apache/linkis/metadata/query/service/oracle/SqlConnection.java rename to linkis-public-enhancements/linkis-datasource/linkis-datasource-manager/service/jdbc/src/main/java/org/apache/linkis/metadata/query/service/oracle/SqlConnection.java diff --git a/linkis-public-enhancements/linkis-datasource/linkis-metadata-query/service/jdbc/src/main/java/org/apache/linkis/metadata/query/service/postgres/SqlConnection.java b/linkis-public-enhancements/linkis-datasource/linkis-datasource-manager/service/jdbc/src/main/java/org/apache/linkis/metadata/query/service/postgres/SqlConnection.java similarity index 100% rename from linkis-public-enhancements/linkis-datasource/linkis-metadata-query/service/jdbc/src/main/java/org/apache/linkis/metadata/query/service/postgres/SqlConnection.java rename to linkis-public-enhancements/linkis-datasource/linkis-datasource-manager/service/jdbc/src/main/java/org/apache/linkis/metadata/query/service/postgres/SqlConnection.java diff --git a/linkis-public-enhancements/linkis-datasource/linkis-metadata-query/service/jdbc/src/main/java/org/apache/linkis/metadata/query/service/sqlserver/SqlConnection.java b/linkis-public-enhancements/linkis-datasource/linkis-datasource-manager/service/jdbc/src/main/java/org/apache/linkis/metadata/query/service/sqlserver/SqlConnection.java similarity index 100% rename from linkis-public-enhancements/linkis-datasource/linkis-metadata-query/service/jdbc/src/main/java/org/apache/linkis/metadata/query/service/sqlserver/SqlConnection.java rename to linkis-public-enhancements/linkis-datasource/linkis-datasource-manager/service/jdbc/src/main/java/org/apache/linkis/metadata/query/service/sqlserver/SqlConnection.java diff --git a/linkis-public-enhancements/linkis-datasource/linkis-metadata-query/service/kafka/pom.xml b/linkis-public-enhancements/linkis-datasource/linkis-datasource-manager/service/kafka/pom.xml similarity index 100% rename from linkis-public-enhancements/linkis-datasource/linkis-metadata-query/service/kafka/pom.xml rename to linkis-public-enhancements/linkis-datasource/linkis-datasource-manager/service/kafka/pom.xml diff --git a/linkis-public-enhancements/linkis-datasource/linkis-metadata-query/service/kafka/src/main/assembly/distribution.xml b/linkis-public-enhancements/linkis-datasource/linkis-datasource-manager/service/kafka/src/main/assembly/distribution.xml similarity index 100% rename from linkis-public-enhancements/linkis-datasource/linkis-metadata-query/service/kafka/src/main/assembly/distribution.xml rename to linkis-public-enhancements/linkis-datasource/linkis-datasource-manager/service/kafka/src/main/assembly/distribution.xml diff --git a/linkis-public-enhancements/linkis-datasource/linkis-metadata-query/service/kafka/src/main/java/org/apache/linkis/metadata/query/service/KafkaConnection.java b/linkis-public-enhancements/linkis-datasource/linkis-datasource-manager/service/kafka/src/main/java/org/apache/linkis/metadata/query/service/KafkaConnection.java similarity index 100% rename from linkis-public-enhancements/linkis-datasource/linkis-metadata-query/service/kafka/src/main/java/org/apache/linkis/metadata/query/service/KafkaConnection.java rename to linkis-public-enhancements/linkis-datasource/linkis-datasource-manager/service/kafka/src/main/java/org/apache/linkis/metadata/query/service/KafkaConnection.java diff --git a/linkis-public-enhancements/linkis-datasource/linkis-metadata-query/service/kafka/src/main/java/org/apache/linkis/metadata/query/service/KafkaMetaService.java b/linkis-public-enhancements/linkis-datasource/linkis-datasource-manager/service/kafka/src/main/java/org/apache/linkis/metadata/query/service/KafkaMetaService.java similarity index 100% rename from linkis-public-enhancements/linkis-datasource/linkis-metadata-query/service/kafka/src/main/java/org/apache/linkis/metadata/query/service/KafkaMetaService.java rename to linkis-public-enhancements/linkis-datasource/linkis-datasource-manager/service/kafka/src/main/java/org/apache/linkis/metadata/query/service/KafkaMetaService.java diff --git a/linkis-public-enhancements/linkis-datasource/linkis-metadata-query/service/kafka/src/main/java/org/apache/linkis/metadata/query/service/KafkaParamsMapper.java b/linkis-public-enhancements/linkis-datasource/linkis-datasource-manager/service/kafka/src/main/java/org/apache/linkis/metadata/query/service/KafkaParamsMapper.java similarity index 100% rename from linkis-public-enhancements/linkis-datasource/linkis-metadata-query/service/kafka/src/main/java/org/apache/linkis/metadata/query/service/KafkaParamsMapper.java rename to linkis-public-enhancements/linkis-datasource/linkis-datasource-manager/service/kafka/src/main/java/org/apache/linkis/metadata/query/service/KafkaParamsMapper.java diff --git a/linkis-public-enhancements/linkis-datasource/linkis-metadata-query/service/mongodb/pom.xml b/linkis-public-enhancements/linkis-datasource/linkis-datasource-manager/service/mongodb/pom.xml similarity index 100% rename from linkis-public-enhancements/linkis-datasource/linkis-metadata-query/service/mongodb/pom.xml rename to linkis-public-enhancements/linkis-datasource/linkis-datasource-manager/service/mongodb/pom.xml diff --git a/linkis-public-enhancements/linkis-datasource/linkis-metadata-query/service/mongodb/src/main/assembly/distribution.xml b/linkis-public-enhancements/linkis-datasource/linkis-datasource-manager/service/mongodb/src/main/assembly/distribution.xml similarity index 100% rename from linkis-public-enhancements/linkis-datasource/linkis-metadata-query/service/mongodb/src/main/assembly/distribution.xml rename to linkis-public-enhancements/linkis-datasource/linkis-datasource-manager/service/mongodb/src/main/assembly/distribution.xml diff --git a/linkis-public-enhancements/linkis-datasource/linkis-metadata-query/service/mongodb/src/main/java/org/apache/linkis/metadata/query/service/MongoDbConnection.java b/linkis-public-enhancements/linkis-datasource/linkis-datasource-manager/service/mongodb/src/main/java/org/apache/linkis/metadata/query/service/MongoDbConnection.java similarity index 97% rename from linkis-public-enhancements/linkis-datasource/linkis-metadata-query/service/mongodb/src/main/java/org/apache/linkis/metadata/query/service/MongoDbConnection.java rename to linkis-public-enhancements/linkis-datasource/linkis-datasource-manager/service/mongodb/src/main/java/org/apache/linkis/metadata/query/service/MongoDbConnection.java index 7a5292852e..3586982bb5 100644 --- a/linkis-public-enhancements/linkis-datasource/linkis-metadata-query/service/mongodb/src/main/java/org/apache/linkis/metadata/query/service/MongoDbConnection.java +++ b/linkis-public-enhancements/linkis-datasource/linkis-datasource-manager/service/mongodb/src/main/java/org/apache/linkis/metadata/query/service/MongoDbConnection.java @@ -177,11 +177,7 @@ public void close() throws IOException { private MongoClient getDBConnection(ConnectMessage connectMessage, String database) throws Exception { - LOG.info( - "mongo information is database:{}, username:{}, passwordd:{} ", - database, - connectMessage.username, - connectMessage.password); + LOG.info("mongo information is database:{}, username:{}", database, connectMessage.username); MongoClient client = null; try { diff --git a/linkis-public-enhancements/linkis-datasource/linkis-metadata-query/service/mongodb/src/main/java/org/apache/linkis/metadata/query/service/MongoDbParamsMapper.java b/linkis-public-enhancements/linkis-datasource/linkis-datasource-manager/service/mongodb/src/main/java/org/apache/linkis/metadata/query/service/MongoDbParamsMapper.java similarity index 100% rename from linkis-public-enhancements/linkis-datasource/linkis-metadata-query/service/mongodb/src/main/java/org/apache/linkis/metadata/query/service/MongoDbParamsMapper.java rename to linkis-public-enhancements/linkis-datasource/linkis-datasource-manager/service/mongodb/src/main/java/org/apache/linkis/metadata/query/service/MongoDbParamsMapper.java diff --git a/linkis-public-enhancements/linkis-datasource/linkis-metadata-query/service/mongodb/src/main/java/org/apache/linkis/metadata/query/service/MongodbMetaService.java b/linkis-public-enhancements/linkis-datasource/linkis-datasource-manager/service/mongodb/src/main/java/org/apache/linkis/metadata/query/service/MongodbMetaService.java similarity index 100% rename from linkis-public-enhancements/linkis-datasource/linkis-metadata-query/service/mongodb/src/main/java/org/apache/linkis/metadata/query/service/MongodbMetaService.java rename to linkis-public-enhancements/linkis-datasource/linkis-datasource-manager/service/mongodb/src/main/java/org/apache/linkis/metadata/query/service/MongodbMetaService.java diff --git a/linkis-public-enhancements/linkis-datasource/linkis-metadata/pom.xml b/linkis-public-enhancements/linkis-datasource/linkis-metadata/pom.xml index 3f683a33f5..9b61302d77 100644 --- a/linkis-public-enhancements/linkis-datasource/linkis-metadata/pom.xml +++ b/linkis-public-enhancements/linkis-datasource/linkis-metadata/pom.xml @@ -26,8 +26,6 @@ linkis-metadata jar - linkis-metadata - org.apache.linkis diff --git a/linkis-public-enhancements/linkis-datasource/pom.xml b/linkis-public-enhancements/linkis-datasource/pom.xml index c7d700a137..b0eae4131a 100644 --- a/linkis-public-enhancements/linkis-datasource/pom.xml +++ b/linkis-public-enhancements/linkis-datasource/pom.xml @@ -28,11 +28,12 @@ pom linkis-datasource-manager/server - linkis-metadata-query/service/elasticsearch - linkis-metadata-query/service/hive - linkis-metadata-query/service/kafka - linkis-metadata-query/service/jdbc - linkis-metadata-query/service/hdfs - linkis-metadata-query/service/mongodb + linkis-datasource-manager/service/elasticsearch + linkis-datasource-manager/service/hive + linkis-datasource-manager/service/kafka + linkis-datasource-manager/service/jdbc + linkis-datasource-manager/service/hdfs + linkis-datasource-manager/service/mongodb + linkis-metadata diff --git a/linkis-public-enhancements/linkis-pes-publicservice/src/main/java/org/apache/linkis/filesystem/restful/api/FsRestfulApi.java b/linkis-public-enhancements/linkis-pes-publicservice/src/main/java/org/apache/linkis/filesystem/restful/api/FsRestfulApi.java index 189ab711fd..0256689e1c 100644 --- a/linkis-public-enhancements/linkis-pes-publicservice/src/main/java/org/apache/linkis/filesystem/restful/api/FsRestfulApi.java +++ b/linkis-public-enhancements/linkis-pes-publicservice/src/main/java/org/apache/linkis/filesystem/restful/api/FsRestfulApi.java @@ -404,6 +404,10 @@ public Message getDirFileTrees( FsPathListWithError fsPathListWithError = fileSystem.listPathWithError(fsPath); if (fsPathListWithError != null) { for (FsPath children : fsPathListWithError.getFsPaths()) { + // parquet and orc compatible, skipping.crc files + if (children.getPath().endsWith(".crc")) { + continue; + } DirFileTree dirFileTreeChildren = new DirFileTree(); dirFileTreeChildren.setName(new File(children.getPath()).getName()); dirFileTreeChildren.setPath(children.getSchemaPath()); diff --git a/linkis-public-enhancements/pom.xml b/linkis-public-enhancements/pom.xml index 760ea05698..7b9385ddb4 100644 --- a/linkis-public-enhancements/pom.xml +++ b/linkis-public-enhancements/pom.xml @@ -34,7 +34,6 @@ linkis-pes-publicservice linkis-bml linkis-context-service - linkis-datasource/linkis-metadata linkis-datasource linkis-udf/linkis-udf-service linkis-jobhistory diff --git a/linkis-spring-cloud-services/linkis-service-gateway/linkis-gateway-core/src/main/scala/org/apache/linkis/gateway/config/GatewayConfiguration.scala b/linkis-spring-cloud-services/linkis-service-gateway/linkis-gateway-core/src/main/scala/org/apache/linkis/gateway/config/GatewayConfiguration.scala index cace679033..9ab75c74ce 100644 --- a/linkis-spring-cloud-services/linkis-service-gateway/linkis-gateway-core/src/main/scala/org/apache/linkis/gateway/config/GatewayConfiguration.scala +++ b/linkis-spring-cloud-services/linkis-service-gateway/linkis-gateway-core/src/main/scala/org/apache/linkis/gateway/config/GatewayConfiguration.scala @@ -110,4 +110,12 @@ object GatewayConfiguration { val LINKIS_CLUSTER_NAME = CommonVars("linkis.cluster.name", "") + val ACCESS_CONTROL_ENABLED = CommonVars("linkis.client.access.control.enable", false) + + val ACCESS_CONTROL_URL = CommonVars("linkis.client.access.control.url", "") + + val ACCESS_CONTROL_IP = CommonVars("linkis.client.access.control.ip", "") + + val ACCESS_CONTROL_USER_ENABLED = CommonVars("linkis.client.access.control.user.enable", false) + } diff --git a/linkis-spring-cloud-services/linkis-service-gateway/linkis-gateway-core/src/main/scala/org/apache/linkis/gateway/security/SecurityFilter.scala b/linkis-spring-cloud-services/linkis-service-gateway/linkis-gateway-core/src/main/scala/org/apache/linkis/gateway/security/SecurityFilter.scala index 2eb458beb2..150ae565ef 100644 --- a/linkis-spring-cloud-services/linkis-service-gateway/linkis-gateway-core/src/main/scala/org/apache/linkis/gateway/security/SecurityFilter.scala +++ b/linkis-spring-cloud-services/linkis-service-gateway/linkis-gateway-core/src/main/scala/org/apache/linkis/gateway/security/SecurityFilter.scala @@ -17,7 +17,7 @@ package org.apache.linkis.gateway.security -import org.apache.linkis.common.conf.Configuration +import org.apache.linkis.common.conf.{CommonVars, Configuration} import org.apache.linkis.common.exception.LinkisException import org.apache.linkis.common.utils.{Logging, Utils} import org.apache.linkis.gateway.config.GatewayConfiguration @@ -43,6 +43,7 @@ object SecurityFilter extends Logging { private val refererValidate = ServerConfiguration.BDP_SERVER_SECURITY_REFERER_VALIDATE.getValue private val referers = ServerConfiguration.BDP_SERVER_ADDRESS.getValue protected val testUser: String = ServerConfiguration.BDP_TEST_USER.getValue + private val ACCESS_CONTROL_USER_PREFIX = "linkis.client.access.control.user." private val ipSet = new util.HashSet[String]() @@ -104,11 +105,12 @@ object SecurityFilter extends Logging { val isPassAuthRequest = GatewayConfiguration.PASS_AUTH_REQUEST_URI.exists(r => !r.equals("") && gatewayContext.getRequest.getRequestURI.startsWith(r) ) - if ( - gatewayContext.getRequest.getRequestURI.startsWith( - ServerConfiguration.BDP_SERVER_USER_URI.getValue - ) - ) { + + val isUserRestful = gatewayContext.getRequest.getRequestURI.startsWith( + ServerConfiguration.BDP_SERVER_USER_URI.getValue + ) + + if (isUserRestful) { Utils.tryCatch(userRestful.doUserRequest(gatewayContext)) { t => val message = t match { case dwc: LinkisException => dwc.getMessage @@ -120,10 +122,9 @@ object SecurityFilter extends Logging { Message.error(message).<<(gatewayContext.getRequest.getRequestURI) ) } - false + return false } else if (isPassAuthRequest && !GatewayConfiguration.ENABLE_SSO_LOGIN.getValue) { logger.info("No login needed for proxy uri: " + gatewayContext.getRequest.getRequestURI) - true } else if (TokenAuthentication.isTokenRequest(gatewayContext)) { TokenAuthentication.tokenAuth(gatewayContext) } else { @@ -142,22 +143,20 @@ object SecurityFilter extends Logging { throw t } if (userName.isDefined) { - true + logger.info(s"User $userName has logged in.") } else if (Configuration.IS_TEST_MODE.getValue) { logger.info("test mode! login for uri: " + gatewayContext.getRequest.getRequestURI) GatewaySSOUtils.setLoginUser(gatewayContext, testUser) - true } else if (GatewayConfiguration.ENABLE_SSO_LOGIN.getValue) { val user = SSOInterceptor.getSSOInterceptor.getUser(gatewayContext) if (StringUtils.isNotBlank(user)) { GatewaySSOUtils.setLoginUser(gatewayContext.getRequest, user) - true } else if (isPassAuthRequest) { gatewayContext.getResponse.redirectTo( SSOInterceptor.getSSOInterceptor.redirectTo(gatewayContext.getRequest.getURI) ) gatewayContext.getResponse.sendResponse() - false + return false } else { filterResponse( gatewayContext, @@ -169,7 +168,7 @@ object SecurityFilter extends Logging { SSOInterceptor.getSSOInterceptor.redirectTo(gatewayContext.getRequest.getURI) ) << gatewayContext.getRequest.getRequestURI ) - false + return false } } else if ( gatewayContext.getRequest.getRequestURI.matches( @@ -179,7 +178,6 @@ object SecurityFilter extends Logging { logger.info( "Not logged in, still let it pass (GATEWAY_NO_AUTH_URL): " + gatewayContext.getRequest.getRequestURI ) - true } else { filterResponse( gatewayContext, @@ -187,9 +185,56 @@ object SecurityFilter extends Logging { "You are not logged in, please login first(您尚未登录，请先登录)!" ) << gatewayContext.getRequest.getRequestURI ) - false + return false + } + } + + // 访问控制, 先判断当前用户是否可以在当前IP执行，再判断当前IP是否有权限调用当前接口 + // Access control + // first determine whether the current user can perform operations from the current IP address, + // and then determine whether the current IP address has permission to call the current interface. + if ( + GatewayConfiguration.ACCESS_CONTROL_USER_ENABLED.getValue && !isPassAuthRequest && !isUserRestful + ) { + val userName = GatewaySSOUtils.getLoginUsername(gatewayContext) + val userIps = + CommonVars.apply(ACCESS_CONTROL_USER_PREFIX + userName, "").getValue + val host = + gatewayContext.getRequest.getRemoteAddress.getAddress.toString.replaceAll("/", "") + if (StringUtils.isNotEmpty(userIps)) { + if (!userIps.contains(host)) { + val message = + Message.error( + s"Unauthorized access! User $userName is prohibited from accessing from the current IP $host. (未授权的访问！用户${userName}禁止在当前IP${host}访问。)" + ) + filterResponse(gatewayContext, message) + return false + } + } + } + if ( + GatewayConfiguration.ACCESS_CONTROL_ENABLED.getValue && !isPassAuthRequest && !isUserRestful + ) { + if ( + StringUtils.isNotEmpty(GatewayConfiguration.ACCESS_CONTROL_IP.getValue) && StringUtils + .isNotEmpty(GatewayConfiguration.ACCESS_CONTROL_URL.getValue) + ) { + val host = + gatewayContext.getRequest.getRemoteAddress.getAddress.toString.replaceAll("/", "") + if (GatewayConfiguration.ACCESS_CONTROL_IP.getValue.contains(host)) { + val requestUrl = gatewayContext.getRequest.getRequestURI + if (!GatewayConfiguration.ACCESS_CONTROL_URL.getValue.contains(requestUrl)) { + val message = + Message.error( + s"Unauthorized access! IP $host is prohibited from accessing this URL. (未授权的访问！当前IP${host}禁止访问此URL。)" + ) + filterResponse(gatewayContext, message) + return false + } + } } } + true } private var userRestful: UserRestful = _ diff --git a/linkis-web-next/package-lock.json b/linkis-web-next/package-lock.json index ff48c85670..ba709e4bda 100644 --- a/linkis-web-next/package-lock.json +++ b/linkis-web-next/package-lock.json @@ -17,7 +17,7 @@ "md5": "^2.3.0", "monaco-editor": "^0.43.0", "qs": "^6.11.2", - "vite": "^4.3.9", + "vite": "^4.4.12", "vue": "^3.2.47", "vue-i18n": "^9.4.1", "vue-router": "^4.2.2" @@ -2840,9 +2840,9 @@ "dev": true }, "node_modules/follow-redirects": { - "version": "1.15.3", - "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.3.tgz", - "integrity": "sha512-1VzOtuEM8pC9SFU1E+8KfTjZyMztRsgEfwQl44z8A25uy13jSzTj6dyK2Df52iV0vgHCfBwLhDWevLn95w5v6Q==", + "version": "1.15.4", + "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.4.tgz", + "integrity": "sha512-Cr4D/5wlrb0z9dgERpUL3LrmPKVDsETIJhaCMeDfuFYcqa5bldGV6wBsAN6X/vxlXQtFBMrXdXxdL8CbDTGniw==", "funding": [ { "type": "individual", @@ -5974,9 +5974,9 @@ } }, "node_modules/vite": { - "version": "4.4.9", - "resolved": "https://registry.npmjs.org/vite/-/vite-4.4.9.tgz", - "integrity": "sha512-2mbUn2LlUmNASWwSCNSJ/EG2HuSRTnVNaydp6vMCm5VIqJsjMfbIWtbH2kDuwUVW5mMUKKZvGPX/rqeqVvv1XA==", + "version": "4.4.12", + "resolved": "https://registry.npmjs.org/vite/-/vite-4.4.12.tgz", + "integrity": "sha512-KtPlUbWfxzGVul8Nut8Gw2Qe8sBzWY+8QVc5SL8iRFnpnrcoCaNlzO40c1R6hPmcdTwIPEDkq0Y9+27a5tVbdQ==", "dependencies": { "esbuild": "^0.18.10", "postcss": "^8.4.27", @@ -8457,9 +8457,9 @@ "dev": true }, "follow-redirects": { - "version": "1.15.3", - "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.3.tgz", - "integrity": "sha512-1VzOtuEM8pC9SFU1E+8KfTjZyMztRsgEfwQl44z8A25uy13jSzTj6dyK2Df52iV0vgHCfBwLhDWevLn95w5v6Q==" + "version": "1.15.4", + "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.4.tgz", + "integrity": "sha512-Cr4D/5wlrb0z9dgERpUL3LrmPKVDsETIJhaCMeDfuFYcqa5bldGV6wBsAN6X/vxlXQtFBMrXdXxdL8CbDTGniw==" }, "for-each": { "version": "0.3.3", @@ -10673,9 +10673,9 @@ } }, "vite": { - "version": "4.4.9", - "resolved": "https://registry.npmjs.org/vite/-/vite-4.4.9.tgz", - "integrity": "sha512-2mbUn2LlUmNASWwSCNSJ/EG2HuSRTnVNaydp6vMCm5VIqJsjMfbIWtbH2kDuwUVW5mMUKKZvGPX/rqeqVvv1XA==", + "version": "4.4.12", + "resolved": "https://registry.npmjs.org/vite/-/vite-4.4.12.tgz", + "integrity": "sha512-KtPlUbWfxzGVul8Nut8Gw2Qe8sBzWY+8QVc5SL8iRFnpnrcoCaNlzO40c1R6hPmcdTwIPEDkq0Y9+27a5tVbdQ==", "requires": { "esbuild": "^0.18.10", "fsevents": "~2.3.2", diff --git a/linkis-web-next/package.json b/linkis-web-next/package.json index b42a2ebe9a..70751d8782 100644 --- a/linkis-web-next/package.json +++ b/linkis-web-next/package.json @@ -19,7 +19,7 @@ "md5": "^2.3.0", "monaco-editor": "^0.43.0", "qs": "^6.11.2", - "vite": "^4.3.9", + "vite": "^4.4.12", "vue": "^3.2.47", "vue-i18n": "^9.4.1", "vue-router": "^4.2.2" diff --git a/linkis-web/.env b/linkis-web/.env index 371b8faf75..d62b9326f3 100644 --- a/linkis-web/.env +++ b/linkis-web/.env @@ -2,4 +2,4 @@ VUE_APP_HOST= BACKEND_URL=http://127.0.0.1:9001 VUE_APP_MN_CONFIG_PREFIX= VUE_APP_MN_CONFIG_SOCKET=/ws/api/entrance/connect -VUE_APP_VERSION=1.4.0 +VUE_APP_VERSION=1.5.0 diff --git a/linkis-web/src/apps/linkis/router.js b/linkis-web/src/apps/linkis/router.js index a94beab0c5..9aa3b05a32 100644 --- a/linkis-web/src/apps/linkis/router.js +++ b/linkis-web/src/apps/linkis/router.js @@ -289,34 +289,34 @@ export default [ }, }, { - name: 'codeQuery', - path: 'codeQuery', + name: 'configManagement', + path: 'configManagement', component: () => - import('./module/codeQuery/index.vue'), + import('./module/configManagement/index.vue'), meta: { - title: 'codeQuery', + title: 'configManagement', publicPage: true, }, }, { - name: 'codeDetail', - path: 'codeDetail', + name: 'viewHistoryDetail', + path: 'viewHistoryDetail', component: () => - import('./module/codeQuery/codeDetail/index.vue'), + import('./module/globalHistoryManagement/viewHistory.vue'), meta: { - title: 'codeDetail', + title: 'viewHistoryDetail', publicPage: true, + noLayout: true }, }, { - name: 'viewHistoryDetail', - path: 'viewHistoryDetail', + name: 'userConfig', + path: 'userConfig', component: () => - import('./module/globalHistoryManagement/viewHistory.vue'), + import('./module/userConfig/index.vue'), meta: { - title: 'viewHistoryDetail', + title: 'userConfig', publicPage: true, - noLayout: true }, }, ], diff --git a/linkis-web/src/apps/linkis/view/linkis/index.vue b/linkis-web/src/apps/linkis/view/linkis/index.vue index fb579a4a0a..07b792801c 100644 --- a/linkis-web/src/apps/linkis/view/linkis/index.vue +++ b/linkis-web/src/apps/linkis/view/linkis/index.vue @@ -180,7 +180,6 @@ export default { ), showSubMenu: true, }, - { key: '1-11', name: this.$t('message.linkis.sideNavList.function.children.codeQuery'), path: '/console/codeQuery' }, { key: '1-12', name: this.$t( @@ -269,13 +268,6 @@ export default { ), path: '/console/ipListManagement', }, - { - key: '1-10-7', - name: this.$t( - 'message.linkis.sideNavList.function.children.acrossClusterRule' - ), - path: '/console/acrossClusterRule', - }, { key: '1-10-8', name: this.$t( @@ -396,8 +388,8 @@ export default { return } if (index === '1-12') { - this.sideNavList.children[10].showSubMenu = - !this.sideNavList.children[10].showSubMenu + this.sideNavList.children[9].showSubMenu = + !this.sideNavList.children[9].showSubMenu return } // index = index.split('-')[0] + '-' + index.split('-')[1]; //防止出现三级菜单 diff --git a/pom.xml b/pom.xml index 2b11f7aafb..9335e2ae19 100644 --- a/pom.xml +++ b/pom.xml @@ -102,7 +102,7 @@ - 1.5.0-SNAPSHOT + 1.5.0 3.5.9 4.2.0 @@ -121,6 +121,9 @@ ${hadoop.version} provided + provided + provided + 1.16.2 0.9.3 1.3.0 @@ -132,6 +135,7 @@ 1 0.234 3.0.0 + 1.2.6 1 python2 2.1.2 @@ -153,6 +157,9 @@ 3.16.3 + 1.10.0 + 1.5.8 + 1.19.4 2.23.1 4.1.86.Final @@ -212,6 +219,12 @@ 2.2.220 + 5.2.23.RELEASE + 5.7.5 + 2.3.12.RELEASE + 2.2.9.RELEASE + Hoxton.SR12 + 2.2.9.RELEASE 5.3.27 5.7.8 2.7.11 @@ -1355,6 +1368,11 @@ + + com.alibaba.cloud + spring-cloud-starter-alibaba-nacos-discovery + ${spring.cloud.nacos.version} + @@ -1934,5 +1952,19 @@ + + + storage-parquet + + compile + + + + + storage-orc + + compile + + diff --git a/tool/dependencies/known-dependencies.txt b/tool/dependencies/known-dependencies.txt index 7346bd4fd9..5954d026a4 100644 --- a/tool/dependencies/known-dependencies.txt +++ b/tool/dependencies/known-dependencies.txt @@ -38,6 +38,8 @@ automaton-1.11-8.jar avatica-1.11.0.jar avro-1.7.7.jar avro-1.8.2.jar +avro-1.10.1.jar +avro-1.10.2.jar aws-java-sdk-core-1.11.792.jar aws-java-sdk-core-1.12.261.jar aws-java-sdk-kms-1.11.792.jar @@ -71,6 +73,7 @@ classgraph-4.1.7.jar classgraph-4.8.83.jar classmate-1.5.1.jar clickhouse-jdbc-0.4.6.jar +client-3.0.0.jar commons-beanutils-1.9.4.jar commons-cli-1.3.1.jar commons-crypto-1.0.0.jar @@ -134,6 +137,7 @@ expiringmap-0.5.6.jar failsafe-2.4.0.jar failureaccess-1.0.1.jar fastutil-6.5.6.jar +fastutil-7.0.13.jar feign-core-10.12.jar feign-core-11.10.jar feign-form-3.8.0.jar @@ -406,6 +410,7 @@ jsr305-3.0.1.jar jsr305-3.0.2.jar jsr311-api-1.1.1.jar jta-1.1.jar +jts-core-1.16.1.jar jul-to-slf4j-1.7.30.jar jul-to-slf4j-1.7.36.jar kafka-clients-2.5.1.jar @@ -579,6 +584,7 @@ opentracing-api-0.33.0.jar opentracing-noop-0.33.0.jar opentracing-util-0.33.0.jar orc-core-1.5.8.jar +orc-core-1.5.8-nohive.jar orc-shims-1.5.8.jar org.jacoco.agent-0.8.5-runtime.jar osgi-resource-locator-1.0.1.jar @@ -587,6 +593,19 @@ pagehelper-5.3.1.jar paranamer-2.3.jar paranamer-2.8.jar parquet-hadoop-bundle-1.10.0.jar +parquet-avro-1.10.0.jar +parquet-column-1.10.0.jar +parquet-column-1.12.2.jar +parquet-common-1.10.0.jar +parquet-common-1.12.2.jar +parquet-encoding-1.10.0.jar +parquet-encoding-1.12.2.jar +parquet-format-2.4.0.jar +parquet-format-structures-1.12.2.jar +parquet-hadoop-1.10.0.jar +parquet-hadoop-1.12.2.jar +parquet-jackson-1.10.0.jar +parquet-jackson-1.12.2.jar poi-5.2.3.jar poi-ooxml-5.2.3.jar poi-ooxml-lite-5.2.3.jar @@ -666,6 +685,8 @@ snakeyaml-1.33.jar snappy-java-1.1.4.jar snappy-java-1.1.7.7.jar snappy-java-1.1.8.2.jar +snappy-java-1.1.8.4.jar +spark-doris-connector-3.2_2.12-1.2.0.jar spark-redis_2.12-2.6.0.jar spring-aop-5.2.23.RELEASE.jar spring-aop-5.3.27.jar @@ -833,12 +854,15 @@ zookeeper-3.5.9.jar zookeeper-jute-3.5.9.jar zstd-jni-1.4.4-7.jar zstd-jni-1.4.5-6.jar +zstd-jni-1.4.9-1.jar +zstd-jni-1.5.0-4.jar zjsonpatch-0.3.0.jar zipkin-2.23.2.jar zipkin-reporter-2.16.3.jar zipkin-reporter-brave-2.16.3.jar zstd-jni-1.5.0-4.jar agrona-1.12.0.jar +audience-annotations-0.12.0.jar audience-annotations-0.13.0.jar commons-crypto-1.1.0.jar disruptor-3.4.2.jar