feat-4787: 1.4.0 feature update

docs/about/configuration.md

@@ -40,9 +40,9 @@ Linkis binary packages are compiled based on the following software versions:
 
 | Component | Version | Description |
 | --- | --- | --- |
-| Hadoop | 2.7.2 | |
-| Hive | 2.3.3 | |
-| Spark | 2.4.3 | |
+| Hadoop | 3.3.4 | |
+| Hive | 3.1.3 | |
+| Spark | 3.2.1 | |
 | Flink | 1.12.2 | |
 | openLooKeng | 1.5.0 | |
 | Sqoop | 1.4.6 | |

docs/deployment/deploy-quick.md

@@ -235,7 +235,7 @@ HADOOP_KEYTAB_PATH=/appcom/keytab/
 >
 > Note: Linkis has not adapted permissions for S3, so it is not possible to grant authorization for it.
 
-`vim linkis.properties`
+`vim $LINKIS_HOME/conf/linkis.properties`
 ```shell script
 # s3 file system
 linkis.storage.s3.access.key=xxx
@@ -245,7 +245,7 @@ linkis.storage.s3.region=xxx
 linkis.storage.s3.bucket=xxx
 ```
 
-`vim linkis-cg-entrance.properties`
+`vim $LINKIS_HOME/conf/linkis-cg-entrance.properties`
 ```shell script
 wds.linkis.entrance.config.log.path=s3:///linkis/logs
 wds.linkis.resultSet.store.path=s3:///linkis/results
@@ -507,10 +507,10 @@ wds.linkis.admin.password= #Password
 sh bin/linkis-cli -submitUser hadoop -engineType shell-1 -codeType shell -code "whoami"
 
 #hive engine tasks
-sh bin/linkis-cli -submitUser hadoop -engineType hive-2.3.3 -codeType hql -code "show tables"
+sh bin/linkis-cli -submitUser hadoop -engineType hive-3.1.3 -codeType hql -code "show tables"
 
 #spark engine tasks
-sh bin/linkis-cli -submitUser hadoop -engineType spark-2.4.3 -codeType sql -code "show tables"
+sh bin/linkis-cli -submitUser hadoop -engineType spark-3.2.1 -codeType sql -code "show tables"
 
 #python engine task
 sh bin/linkis-cli -submitUser hadoop -engineType python-python2 -codeType python -code 'print("hello, world!")'
@@ -551,9 +551,9 @@ $ tree linkis-package/lib/linkis-engineconn-plugins/ -L 3
 linkis-package/lib/linkis-engineconn-plugins/
 ├── hive
 │ ├── dist
-│ │ └── 2.3.3 #version is 2.3.3 engineType is hive-2.3.3
+│ │ └── 3.1.3 #version is 3.1.3 engineType is hive-3.1.3
 │ └── plugin
-│ └── 2.3.3
+│ └── 3.1.3
 ├── python
 │ ├── dist
 │ │ └── python2
@@ -566,9 +566,9 @@ linkis-package/lib/linkis-engineconn-plugins/
 │ └── 1
 └── spark
     ├── dist
-    │ └── 2.4.3
+    │ └── 3.2.1
     └── plugin
-        └── 2.4.3
+        └── 3.2.1
 ````
 
 #### Method 2: View the database table of linkis
@@ -595,13 +595,13 @@ Insert yarn data information
 INSERT INTO `linkis_cg_rm_external_resource_provider`
 (`resource_type`, `name`, `labels`, `config`) VALUES
 ('Yarn', 'sit', NULL,
-'{\r\n"rmWebAddress": "http://xx.xx.xx.xx:8088",\r\n"hadoopVersion": "2.7.2",\r\n"authorEnable":false, \r\n"user":"hadoop",\r\n"pwd":"123456"\r\n}'
+'{\r\n"rmWebAddress": "http://xx.xx.xx.xx:8088",\r\n"hadoopVersion": "3.3.4",\r\n"authorEnable":false, \r\n"user":"hadoop",\r\n"pwd":"123456"\r\n}'
 );
 
 config field properties
 
 "rmWebAddress": "http://xx.xx.xx.xx:8088", #need to bring http and port
-"hadoopVersion": "2.7.2",
+"hadoopVersion": "3.3.4",
 "authorEnable":true, //Whether authentication is required You can verify the username and password by visiting http://xx.xx.xx.xx:8088 in the browser
 "user":"user",//username
 "pwd":"pwd"//Password

docs/engine-usage/hive.md

@@ -42,7 +42,7 @@ The version of `Hive` supports `hive1.x` and `hive2.x`. The default is to suppor
 
 <https://github.com/apache/linkis/pull/541>
 
-The `hive` version supported by default is 2.3.3, if you want to modify the `hive` version, you can find the `linkis-engineplugin-hive` module, modify the \<hive.version\> tag, and then compile this module separately Can
+The `hive` version supported by default is 3.1.3, if you want to modify the `hive` version, you can find the `linkis-engineplugin-hive` module, modify the \<hive.version\> tag, and then compile this module separately Can
 
 [EngineConnPlugin engine plugin installation](../deployment/install-engineconn.md)
 
@@ -51,7 +51,7 @@ The `hive` version supported by default is 2.3.3, if you want to modify the `hiv
 ### 3.1 Submitting tasks via `Linkis-cli`
 
 ```shell
-sh ./bin/linkis-cli -engineType hive-2.3.3 \
+sh ./bin/linkis-cli -engineType hive-3.1.3 \
 -codeType hql -code "show databases"  \
 -submitUser hadoop -proxyUser hadoop
 ```
@@ -65,7 +65,7 @@ For the `Hive` task, you only need to modify `EngineConnType` and `CodeType` par
 
 ```java
 Map<String, Object> labels = new HashMap<String, Object>();
-labels.put(LabelKeyConstant.ENGINE_TYPE_KEY, "hive-2.3.3"); // required engineType Label
+labels.put(LabelKeyConstant.ENGINE_TYPE_KEY, "hive-3.1.3"); // required engineType Label
 labels.put(LabelKeyConstant.USER_CREATOR_TYPE_KEY, "hadoop-IDE");// required execute user and creator
 labels.put(LabelKeyConstant.CODE_TYPE_KEY, "hql"); // required codeType
 ```
@@ -95,7 +95,7 @@ Note: After modifying the configuration under the `IDE` tag, you need to specify
 
 ```shell
 sh ./bin/linkis-cli -creator IDE \
--engineType hive-2.3.3 -codeType hql \
+-engineType hive-3.1.3 -codeType hql \
 -code "show databases"  \
 -submitUser hadoop -proxyUser hadoop
 ```
@@ -116,7 +116,7 @@ Example of http request parameters
                             }
                     },
     "labels": {
-        "engineType": "hive-2.3.3",
+        "engineType": "hive-3.1.3",
         "userCreator": "hadoop-IDE"
     }
 }
@@ -128,7 +128,7 @@ Example of http request parameters
 
 ```
 linkis_ps_configuration_config_key: Insert the key and default values ​​​​of the configuration parameters of the engine
-linkis_cg_manager_label: insert engine label such as: hive-2.3.3
+linkis_cg_manager_label: insert engine label such as: hive-3.1.3
 linkis_ps_configuration_category: Insert the directory association of the engine
 linkis_ps_configuration_config_value: The configuration that the insertion engine needs to display
 linkis_ps_configuration_key_engine_relation: The relationship between the configuration item and the engine
@@ -138,7 +138,7 @@ The initial data related to the engine in the table is as follows
 
 ```sql
 -- set variable
-SET @HIVE_LABEL="hive-2.3.3";
+SET @HIVE_LABEL="hive-3.1.3";
 SET @HIVE_ALL=CONCAT('*-*,',@HIVE_LABEL);
 SET @HIVE_IDE=CONCAT('*-IDE,',@HIVE_LABEL);
 

docs/engine-usage/impala.md

@@ -1,42 +1,32 @@
 ---
 title: Impala
-sidebar_position: 15
+sidebar_position: 12
 ---
 
 This article mainly introduces the installation, usage and configuration of the `Impala` engine plugin in `Linkis`.
 
-
 ## 1. Pre-work
 
-### 1.1 Engine installation
+### 1.1 Environment installation
 
-If you want to use the `Impala` engine on your `Linkis` service, you need to prepare the Impala service and provide connection information, such as the connection address of the Impala cluster, SASL username and password, etc.
+If you want to use the Impala engine on your server, you need to prepare the Impala service and provide connection information, such as the connection address of the Impala cluster, SASL user name and password, etc.
 
-### 1.2 Service Verification
+### 1.2 Environment verification
 
-```shell
-# prepare trino-cli
-wget https://repo1.maven.org/maven2/io/trino/trino-cli/374/trino-cli-374-executable.jar
-mv trill-cli-374-executable.jar trill-cli
-chmod +x trino-cli
-
-# Execute the task
-./trino-cli --server localhost:8080 --execute 'show tables from system.jdbc'
-
-# Get the following output to indicate that the service is available
-"attributes"
-"catalogs"
-"columns"
-"procedure_columns"
-"procedures"
-"pseudo_columns"
-"schemas"
-"super_tables"
-"super_types"
-"table_types"
-"tables"
-"types"
-"udts"
+Execute the impala-shell command to get the following output, indicating that the impala service is available.
+```
+[root@8f43473645b1 /]# impala-shell
+Starting Impala Shell without Kerberos authentication
+Connected to 8f43473645b1:21000
+Server version: impalad version 2.12.0-cdh5.15.0 RELEASE (build 23f574543323301846b41fa5433690df32efe085)
+***************************************************** *********************************
+Welcome to the Impala shell.
+(Impala Shell v2.12.0-cdh5.15.0 (23f5745) built on Thu May 24 04:07:31 PDT 2018)
+
+When pretty-printing is disabled, you can use the '--output_delimiter' flag to set
+the delimiter for fields in the same row. The default is ','.
+***************************************************** *********************************
+[8f43473645b1:21000] >
 ```
 
 ## 2. Engine plugin deployment
@@ -101,7 +91,7 @@ select * from linkis_cg_engine_conn_plugin_bml_resources;
 
 ```shell
 sh ./bin/linkis-cli -submitUser impala \
--engineType impala-3.4.0 -code 'select * from default.test limit 10' \
+-engineType impala-3.4.0 -code 'show databases;' \
 -runtimeMap linkis.es.http.method=GET \
 -runtimeMap linkis.impala.servers=127.0.0.1:21050
 ```
@@ -143,37 +133,23 @@ More `Linkis-Cli` command parameter reference: [Linkis-Cli usage](../user-guide/
 
 If the default parameters are not satisfied, there are the following ways to configure some basic parameters
 
-#### 4.2.1 Management console configuration
-
-![](./images/trino-config.png)
-
-Note: After modifying the configuration under the `IDE` tag, you need to specify `-creator IDE` to take effect (other tags are similar), such as:
-
-```shell
-sh ./bin/linkis-cli -creator IDE -submitUser hadoop \
- -engineType impala-3.4.0 -codeType sql \
- -code 'select * from system.jdbc.schemas limit 10' 
-```
-
-#### 4.2.2 Task interface configuration
+#### 4.2.1 Task interface configuration
 Submit the task interface and configure it through the parameter `params.configuration.runtime`
 
 ```shell
 Example of http request parameters
 {
-    "executionContent": {"code": "select * from system.jdbc.schemas limit 10;", "runType":  "sql"},
+    "executionContent": {"code": "show databases;", "runType":  "sql"},
     "params": {
                     "variable": {},
                     "configuration": {
                             "runtime": {
-                                "linkis.trino.url":"http://127.0.0.1:8080",
-                                "linkis.trino.catalog ":"hive",
-                                "linkis.trino.schema ":"default"
-                                }
+                                "linkis.impala.servers"="127.0.0.1:21050"
                             }
-                    },
+                    }
+                },
     "labels": {
-        "engineType": "trino-371",
+        "engineType": "impala-3.4.0",
         "userCreator": "hadoop-IDE"
     }
 }
@@ -185,7 +161,7 @@ Example of http request parameters
 
 ```
 linkis_ps_configuration_config_key: Insert the key and default values ​​​​of the configuration parameters of the engine
-linkis_cg_manager_label: insert engine label such as: trino-375
+linkis_cg_manager_label: insert engine label such as: impala-3.4.0
 linkis_ps_configuration_category: Insert the directory association of the engine
 linkis_ps_configuration_config_value: Insert the configuration that the engine needs to display
 linkis_ps_configuration_key_engine_relation: the relationship between configuration items and engines

docs/engine-usage/jdbc.md

@@ -202,11 +202,7 @@ Note: After modifying the configuration under the `IDE` tag, you need to specify
 sh ./bin/linkis-cli -creator IDE \
 -engineType jdbc-4 -codeType jdbc \
 -code "show tables"  \
--submitUser hadoop -proxyUser hadoop \
--runtimeMap wds.linkis.jdbc.connect.url=jdbc:mysql://127.0.0.1:3306 \
--runtimeMap wds.linkis.jdbc.driver=com.mysql.jdbc.Driver \
--runtimeMap wds.linkis.jdbc.username=root \
--runtimeMap wds.linkis.jdbc.password=123456 \
+-submitUser hadoop -proxyUser hadoop 
 ```
 
 #### 4.2.2 Task interface configuration

docs/engine-usage/overview.md

@@ -14,8 +14,8 @@ Supported engines and version information are as follows:
 
 | Engine | Default Engine | Default Version |
 |--------------| -- | ---- |
-| [Spark](./spark.md) | Yes | 2.4.3 |
-| [Hive](./hive.md) | yes | 2.3.3 |
+| [Spark](./spark.md) | Yes | 3.2.1 |
+| [Hive](./hive.md) | yes | 3.1.3 |
 | [Python](./python.md) | yes | python2 |
 | [Shell](./shell.md) | Yes | 1 |
 | [JDBC](./jdbc.md) | No | 4 |

docs/engine-usage/spark.md

@@ -32,7 +32,7 @@ Welcome to
       ______
      /__/__ ___ _____/ /__
     _\ \/ _ \/ _ `/ __/ '_/
-   /__ / .__/\_,_/_/ /_/\_\   version 2.4.3
+   /__ / .__/\_,_/_/ /_/\_\   version 3.2.1
       /_/
 
 Using Python version 2.7.13 (default, Sep 30 2017 18:12:43)
@@ -43,7 +43,7 @@ SparkSession available as 'spark'.
 
 The `Spark` engine plugin is included in the binary installation package released by `linkis` by default, and users do not need to install it additionally.
 
-In theory `Linkis` supports all versions of `spark2.x` and above. The default supported version is `Spark2.4.3`. If you want to use another version of `spark`, such as `spark2.1.0`, you just need to modify the version of the plugin `spark` and compile it. Specifically, you can find the `linkis-engineplugin-spark` module, change the value of the `<spark.version>` tag in the `maven` dependency to 2.1.0, and then compile this module separately.
+In theory `Linkis` supports all versions of `spark2.x` and above. The default supported version is `Spark3.2.1`. If you want to use another version of `spark`, such as `spark2.1.0`, you just need to modify the version of the plugin `spark` and compile it. Specifically, you can find the `linkis-engineplugin-spark` module, change the value of the `<spark.version>` tag in the `maven` dependency to 2.1.0, and then compile this module separately.
 
 [EngineConnPlugin engine plugin installation](../deployment/install-engineconn.md)
 
@@ -53,10 +53,10 @@ In theory `Linkis` supports all versions of `spark2.x` and above. The default su
 
 ```shell
 # codeType correspondence py-->pyspark sql-->sparkSQL scala-->Spark scala
-sh ./bin/linkis-cli -engineType spark-2.4.3 -codeType sql -code "show databases"  -submitUser hadoop -proxyUser hadoop
+sh ./bin/linkis-cli -engineType spark-3.2.1 -codeType sql -code "show databases"  -submitUser hadoop -proxyUser hadoop
 
 # You can specify the yarn queue in the submission parameter by -confMap wds.linkis.yarnqueue=dws
-sh ./bin/linkis-cli -engineType spark-2.4.3 -codeType sql  -confMap wds.linkis.yarnqueue=dws -code "show databases"  -submitUser hadoop -proxyUser hadoop
+sh ./bin/linkis-cli -engineType spark-3.2.1 -codeType sql  -confMap wds.linkis.yarnqueue=dws -code "show databases"  -submitUser hadoop -proxyUser hadoop
 ```
 More `Linkis-Cli` command parameter reference: [Linkis-Cli usage](../user-guide/linkiscli-manual.md)
 
@@ -67,7 +67,7 @@ For `Spark` tasks you only need to modify the `EngineConnType` and `CodeType` pa
 
 ```java
 Map<String, Object> labels = new HashMap<String, Object>();
-labels.put(LabelKeyConstant.ENGINE_TYPE_KEY, "spark-2.4.3"); // required engineType Label
+labels.put(LabelKeyConstant.ENGINE_TYPE_KEY, "spark-3.2.1"); // required engineType Label
 labels.put(LabelKeyConstant.USER_CREATOR_TYPE_KEY, "hadoop-IDE");// required execute user and creator
 labels.put(LabelKeyConstant.CODE_TYPE_KEY, "sql"); // required codeType py,sql,scala
 ```
@@ -171,7 +171,7 @@ Token-User: linkis
     },
     "labels": {
         // pattern：engineType-version
-        "engineType": "spark-2.4.3",
+        "engineType": "spark-3.2.1",
         // userCreator: linkis is username。IDE is system that be configed in Linkis。
         "userCreator": "linkis-IDE"
     }
@@ -207,7 +207,7 @@ Note: After modifying the configuration under the `IDE` tag, you need to specify
 
 ```shell
 sh ./bin/linkis-cli -creator IDE \
--engineType spark-2.4.3 -codeType sql \
+-engineType spark-3.2.1 -codeType sql \
 -code "show databases"  \
 -submitUser hadoop -proxyUser hadoop
 ```
@@ -228,7 +228,7 @@ Example of http request parameters
                             }
                     },
     "labels": {
-        "engineType": "spark-2.4.3",
+        "engineType": "spark-3.2.1",
         "userCreator": "hadoop-IDE"
     }
 }
@@ -240,7 +240,7 @@ Example of http request parameters
 
 ```
 linkis_ps_configuration_config_key: Insert the key and default values ​​​​of the configuration parameters of the engine
-linkis_cg_manager_label: insert engine label such as: spark-2.4.3
+linkis_cg_manager_label: insert engine label such as: spark-3.2.1
 linkis_ps_configuration_category: The directory association relationship of the insertion engine
 linkis_ps_configuration_config_value: The configuration that the insertion engine needs to display
 linkis_ps_configuration_key_engine_relation: The relationship between the configuration item and the engine
@@ -250,7 +250,7 @@ The initial data in the table related to the `spark` engine is as follows
 
 ```sql
 -- set variable
-SET @SPARK_LABEL="spark-2.4.3";
+SET @SPARK_LABEL="spark-3.2.1";
 SET @SPARK_ALL=CONCAT('*-*,',@SPARK_LABEL);
 SET @SPARK_IDE=CONCAT('*-IDE,',@SPARK_LABEL);