Merge branch 'master' into master-spring

# Conflicts: # pom.xml
apache · Jan 25, 2024 · 4e5b39e · 4e5b39e
2 parents c41f3f1 + 20c2654
commit 4e5b39e
Show file tree

Hide file tree

Showing 231 changed files with 6,990 additions and 1,023 deletions.
diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md
@@ -12,7 +12,7 @@ and session management.
 
 ### Related issues/PRs
 
-Related issues: #590
+Related issues: close #590 close #591
 Related pr:#591
 
 

diff --git a/.github/workflows/integration-test.yml b/.github/workflows/integration-test.yml
@@ -45,7 +45,7 @@ jobs:
       TAG: ${{ github.sha }}
       SKIP_TEST: true
       HUB: ghcr.io/apache/linkis
-      LINKIS_VERSION: 1.5.0-SNAPSHOT
+      LINKIS_VERSION: 1.5.0
     steps:
       - name: Free up disk space
         run: |

diff --git a/.github/workflows/publish-docker.yaml b/.github/workflows/publish-docker.yaml
@@ -34,7 +34,7 @@ jobs:
       TAG: ${{ github.sha }}
       SKIP_TEST: true
       HUB: ghcr.io/apache/linkis
-      LINKIS_VERSION: 1.5.0-SNAPSHOT
+      LINKIS_VERSION: 1.5.0
     steps:
     - name: Checkout
       uses: actions/checkout@v2

diff --git a/.github/workflows/publish-snapshot.yml b/.github/workflows/publish-snapshot.yml
@@ -29,7 +29,7 @@ jobs:
       fail-fast: false
       matrix:
         branch:
-          - dev-1.4.0
+          - dev-1.5.0
     steps:
       - name: Checkout repository
         uses: actions/checkout@v2

diff --git a/.../linkis-common/src/test/java/org/apache/linkis/common/exception/ExceptionManagerTest.java b/.../linkis-common/src/test/java/org/apache/linkis/common/exception/ExceptionManagerTest.java
@@ -21,6 +21,7 @@
 
 import java.util.HashMap;
 import java.util.Map;
+import java.util.TreeMap;
 
 import org.junit.jupiter.api.Test;
 
@@ -40,7 +41,7 @@ void testGenerateException() {
                 + "null");
     assertEquals(errorException.getClass(), ExceptionManager.generateException(null).getClass());
     assertEquals(errorException.toString(), ExceptionManager.generateException(null).toString());
-    Map<String, Object> map = new HashMap<>();
+    Map<String, Object> map = new TreeMap<>();
     map.put("level", null);
     map.put("errCode", 1);
     map.put("desc", "test");

diff --git a/linkis-commons/linkis-module/pom.xml b/linkis-commons/linkis-module/pom.xml
@@ -64,10 +64,6 @@
         </exclusion>
       </exclusions>
     </dependency>
-    <dependency>
-      <groupId>org.springframework.cloud</groupId>
-      <artifactId>spring-cloud-starter-netflix-eureka-client</artifactId>
-    </dependency>
 
     <dependency>
       <groupId>org.springframework.boot</groupId>
@@ -281,4 +277,80 @@
     </plugins>
   </build>
 
+  <profiles>
+    <profile>
+      <id>eureka</id>
+      <activation>
+        <activeByDefault>true</activeByDefault>
+        <property>
+          <name>discovery</name>
+          <value>eureka</value>
+        </property>
+      </activation>
+      <dependencies>
+        <dependency>
+          <groupId>org.springframework.cloud</groupId>
+          <artifactId>spring-cloud-starter-netflix-eureka-client</artifactId>
+        </dependency>
+      </dependencies>
+    </profile>
+    <profile>
+      <id>nacos</id>
+      <activation>
+        <property>
+          <name>discovery</name>
+          <value>nacos</value>
+        </property>
+      </activation>
+      <dependencies>
+        <dependency>
+          <groupId>com.alibaba.cloud</groupId>
+          <artifactId>spring-cloud-starter-alibaba-nacos-discovery</artifactId>
+          <exclusions>
+            <exclusion>
+              <groupId>org.springframework.boot</groupId>
+              <artifactId>*</artifactId>
+            </exclusion>
+            <exclusion>
+              <groupId>org.springframework.cloud</groupId>
+              <artifactId>spring-cloud-commons</artifactId>
+            </exclusion>
+            <exclusion>
+              <groupId>org.springframework.cloud</groupId>
+              <artifactId>spring-cloud-context</artifactId>
+            </exclusion>
+            <exclusion>
+              <groupId>org.springframework.boot</groupId>
+              <artifactId>spring-boot-starter</artifactId>
+            </exclusion>
+            <exclusion>
+              <groupId>com.fasterxml.jackson.core</groupId>
+              <artifactId>jackson-core</artifactId>
+            </exclusion>
+            <exclusion>
+              <groupId>com.fasterxml.jackson.core</groupId>
+              <artifactId>jackson-databind</artifactId>
+            </exclusion>
+            <exclusion>
+              <groupId>com.google.code.findbugs</groupId>
+              <artifactId>jsr305</artifactId>
+            </exclusion>
+            <exclusion>
+              <groupId>org.yaml</groupId>
+              <artifactId>snakeyaml</artifactId>
+            </exclusion>
+            <exclusion>
+              <groupId>io.prometheus</groupId>
+              <artifactId>simpleclient</artifactId>
+            </exclusion>
+            <exclusion>
+              <groupId>com.google.guava</groupId>
+              <artifactId>guava</artifactId>
+            </exclusion>
+          </exclusions>
+        </dependency>
+      </dependencies>
+    </profile>
+  </profiles>
+
 </project>
diff --git a/linkis-commons/linkis-storage/pom.xml b/linkis-commons/linkis-storage/pom.xml
@@ -99,6 +99,52 @@
       <artifactId>aws-java-sdk-s3</artifactId>
       <version>1.12.261</version>
     </dependency>
+
+    <dependency>
+      <groupId>org.apache.parquet</groupId>
+      <artifactId>parquet-avro</artifactId>
+      <version>${parquet-avro.version}</version>
+      <scope>${storage.parquet.scope}</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.hadoop</groupId>
+      <artifactId>hadoop-mapreduce-client-core</artifactId>
+      <version>${hadoop.version}</version>
+      <scope>${storage.parquet.scope}</scope>
+      <exclusions>
+        <exclusion>
+          <groupId>log4j</groupId>
+          <artifactId>log4j</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>org.slf4j</groupId>
+          <artifactId>slf4j-log4j12</artifactId>
+        </exclusion>
+        <!-- for hadoop 3.3.3 -->
+        <exclusion>
+          <groupId>ch.qos.reload4j</groupId>
+          <artifactId>reload4j</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>org.slf4j</groupId>
+          <artifactId>slf4j-reload4j</artifactId>
+        </exclusion>
+      </exclusions>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.orc</groupId>
+      <artifactId>orc-core</artifactId>
+      <version>${orc-core.version}</version>
+      <classifier>nohive</classifier>
+      <scope>${storage.orc.scope}</scope>
+      <exclusions>
+        <exclusion>
+          <groupId>org.apache.hive</groupId>
+          <artifactId>hive-storage-api</artifactId>
+        </exclusion>
+      </exclusions>
+    </dependency>
+
   </dependencies>
 
   <build>

diff --git a/...ommons/linkis-storage/src/main/java/org/apache/linkis/storage/conf/LinkisStorageConf.java b/...ommons/linkis-storage/src/main/java/org/apache/linkis/storage/conf/LinkisStorageConf.java
@@ -25,6 +25,16 @@
 public class LinkisStorageConf {
   private static final Object CONF_LOCK = new Object();
 
+  public static final String DOLPHIN = "dolphin";
+
+  public static final String PARQUET = "parquet";
+
+  public static final String PARQUET_FILE_SUFFIX = ".parquet";
+
+  public static final String ORC = "orc";
+
+  public static final String ORC_FILE_SUFFIX = ".orc";
+
   public static final String HDFS_FILE_SYSTEM_REST_ERRS =
       CommonVars.apply(
               "wds.linkis.hdfs.rest.errs",
@@ -34,12 +44,19 @@ public class LinkisStorageConf {
   public static final String ROW_BYTE_MAX_LEN_STR =
       CommonVars.apply("wds.linkis.resultset.row.max.str", "2m").getValue();
 
+  public static final String ENGINE_RESULT_TYPE =
+      CommonVars.apply("linkis.engine.resultSet.type", DOLPHIN, "Result type").getValue();
+
   public static final long ROW_BYTE_MAX_LEN = ByteTimeUtils.byteStringAsBytes(ROW_BYTE_MAX_LEN_STR);
 
   public static final String FILE_TYPE =
       CommonVars.apply(
               "wds.linkis.storage.file.type",
-              "dolphin,sql,scala,py,hql,python,out,log,text,txt,sh,jdbc,ngql,psql,fql,tsql")
+              "dolphin,sql,scala,py,hql,python,out,log,text,txt,sh,jdbc,ngql,psql,fql,tsql"
+                  + ","
+                  + PARQUET
+                  + ","
+                  + ORC)
           .getValue();
 
   private static volatile String[] fileTypeArr = null;

diff --git a/...ns/linkis-storage/src/main/java/org/apache/linkis/storage/exception/StorageErrorCode.java b/...ns/linkis-storage/src/main/java/org/apache/linkis/storage/exception/StorageErrorCode.java
@@ -23,7 +23,13 @@ public enum StorageErrorCode {
   FS_NOT_INIT(53001, "please init first"),
 
   INCONSISTENT_DATA(53001, "Inconsistent row data read,read %s,need rowLen %s"),
-  FS_OOM(53002, "OOM occurred while reading the file");
+  FS_OOM(53002, "OOM occurred while reading the file"),
+
+  FS_ERROR(53003, "Failed to operation fs"),
+
+  READ_PARQUET_FAILED(53004, "Failed to read parquet file"),
+
+  READ_ORC_FAILED(53005, "Failed to read orc file");
 
   StorageErrorCode(int errorCode, String message) {
     this.code = errorCode;

diff --git a/...inkis-storage/src/main/java/org/apache/linkis/storage/exception/StorageReadException.java b/...inkis-storage/src/main/java/org/apache/linkis/storage/exception/StorageReadException.java
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.linkis.storage.exception;
+
+import org.apache.linkis.common.exception.ErrorException;
+
+public class StorageReadException extends ErrorException {
+
+  public StorageReadException(int errCode, String desc) {
+    super(errCode, desc);
+  }
+
+  public StorageReadException(int errCode, String desc, Throwable t) {
+    super(errCode, desc);
+    initCause(t);
+  }
+
+  public StorageReadException(int errCode, String desc, String ip, int port, String serviceKind) {
+    super(errCode, desc, ip, port, serviceKind);
+  }
+}