Skip to content

Commit

Permalink
Code optimization
Browse files Browse the repository at this point in the history
  • Loading branch information
“v_kkhuang” committed Oct 31, 2024
1 parent 81d9cae commit d997a03
Show file tree
Hide file tree
Showing 5 changed files with 56 additions and 20 deletions.
16 changes: 16 additions & 0 deletions ...on/src/main/java/org/apache/linkis/configuration/restful/api/ConfigurationRestfulApi.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -570,6 +570,22 @@ public Message saveKeyValue(HttpServletRequest req, @RequestBody Map<String, Obj
return Message.error(e.getMessage());
}
}
if (AESUtils.LINKIS_DATASOURCE_AES_SWITCH.getValue()
&& (configKeyValue.getKey().equals("linkis.nebula.password")
|| configKeyValue.getKey().equals("wds.linkis.jdbc.password"))
&& StringUtils.isNotBlank(configKeyValue.getConfigValue())) {
List<ConfigUserValue> userConfigValue =
configKeyService.getUserConfigValue(engineType, configKeyValue.getKey(), creator, user);
for (ConfigUserValue configUserValue : userConfigValue) {
if ((configUserValue.getKey().equals("linkis.nebula.password")
|| configUserValue.getKey().equals("wds.linkis.jdbc.password"))
&& !configUserValue.getConfigValue().equals(configKeyValue.getConfigValue())) {
configKeyValue.setConfigValue(
AESUtils.encrypt(
configKeyValue.getConfigValue(), AESUtils.LINKIS_DATASOURCE_AES_KEY.getValue()));
}
}
}
ConfigValue configValue = configKeyService.saveConfigValue(configKeyValue, labelList);
configurationService.clearAMCacheConf(username, creator, engineType, version);
return message.data("configValue", configValue);
Expand Down
28 changes: 19 additions & 9 deletions .../main/java/org/apache/linkis/datasourcemanager/core/restful/DataSourceCoreRestfulApi.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -389,7 +389,9 @@ public Message getInfoByDataSourceId(
List<DataSourceParamKeyDefinition> keyDefinitionList =
dataSourceRelateService.getKeyDefinitionsByType(dataSource.getDataSourceTypeId());
// Decrypt
RestfulApiHelper.decryptPasswordKey(keyDefinitionList, dataSource.getConnectParams());
if (!AESUtils.LINKIS_DATASOURCE_AES_SWITCH.getValue()) {
RestfulApiHelper.decryptPasswordKey(keyDefinitionList, dataSource.getConnectParams());
}
return Message.ok().data("info", dataSource);
},
"Fail to access data source[获取数据源信息失败]");
Expand Down Expand Up @@ -425,8 +427,9 @@ public Message getInfoByDataSourceName(
List<DataSourceParamKeyDefinition> keyDefinitionList =
dataSourceRelateService.getKeyDefinitionsByType(dataSource.getDataSourceTypeId());
// Decrypt
RestfulApiHelper.decryptPasswordKey(keyDefinitionList, dataSource.getConnectParams());

if (!AESUtils.LINKIS_DATASOURCE_AES_SWITCH.getValue()) {
RestfulApiHelper.decryptPasswordKey(keyDefinitionList, dataSource.getConnectParams());
}
return Message.ok().data("info", dataSource);
},
"Fail to access data source[获取数据源信息失败]");
Expand Down Expand Up @@ -461,8 +464,9 @@ public Message getPublishedInfoByDataSourceName(
List<DataSourceParamKeyDefinition> keyDefinitionList =
dataSourceRelateService.getKeyDefinitionsByType(dataSource.getDataSourceTypeId());
// Decrypt
RestfulApiHelper.decryptPasswordKey(keyDefinitionList, dataSource.getConnectParams());

if (!AESUtils.LINKIS_DATASOURCE_AES_SWITCH.getValue()) {
RestfulApiHelper.decryptPasswordKey(keyDefinitionList, dataSource.getConnectParams());
}
return Message.ok().data("info", dataSource);
},
"Fail to access data source[获取数据源信息失败]");
Expand Down Expand Up @@ -506,7 +510,9 @@ public Message getInfoByDataSourceIdAndVersion(
List<DataSourceParamKeyDefinition> keyDefinitionList =
dataSourceRelateService.getKeyDefinitionsByType(dataSource.getDataSourceTypeId());
// Decrypt
RestfulApiHelper.decryptPasswordKey(keyDefinitionList, dataSource.getConnectParams());
if (!AESUtils.LINKIS_DATASOURCE_AES_SWITCH.getValue()) {
RestfulApiHelper.decryptPasswordKey(keyDefinitionList, dataSource.getConnectParams());
}
return Message.ok().data("info", dataSource);
},
"Fail to access data source[获取数据源信息失败]");
Expand Down Expand Up @@ -544,7 +550,7 @@ public Message getVersionList(
List<DataSourceParamKeyDefinition> keyDefinitionList =
dataSourceRelateService.getKeyDefinitionsByType(dataSource.getDataSourceTypeId());
// Decrypt
if (null != versions) {
if (!AESUtils.LINKIS_DATASOURCE_AES_SWITCH.getValue() && null != versions) {
versions.forEach(
version -> {
RestfulApiHelper.decryptPasswordKey(
Expand Down Expand Up @@ -694,7 +700,9 @@ public Message getConnectParams(
Map<String, Object> connectParams = dataSource.getConnectParams();
List<DataSourceParamKeyDefinition> keyDefinitionList =
dataSourceRelateService.getKeyDefinitionsByType(dataSource.getDataSourceTypeId());
RestfulApiHelper.decryptPasswordKey(keyDefinitionList, connectParams);
if (!AESUtils.LINKIS_DATASOURCE_AES_SWITCH.getValue()) {
RestfulApiHelper.decryptPasswordKey(keyDefinitionList, connectParams);
}
return Message.ok().data("connectParams", connectParams);
},
"Fail to connect data source[连接数据源失败]");
Expand Down Expand Up @@ -730,7 +738,9 @@ public Message getConnectParams(

List<DataSourceParamKeyDefinition> keyDefinitionList =
dataSourceRelateService.getKeyDefinitionsByType(dataSource.getDataSourceTypeId());
RestfulApiHelper.decryptPasswordKey(keyDefinitionList, connectParams);
if (!AESUtils.LINKIS_DATASOURCE_AES_SWITCH.getValue()) {
RestfulApiHelper.decryptPasswordKey(keyDefinitionList, connectParams);
}
return Message.ok().data("connectParams", connectParams);
},
"Fail to connect data source[连接数据源失败]");
Expand Down
21 changes: 16 additions & 5 deletions ...es-publicservice/src/main/java/org/apache/linkis/filesystem/restful/api/FsRestfulApi.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,7 @@
import org.apache.linkis.storage.source.FileSource$;
import org.apache.linkis.storage.utils.StorageUtils;

import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.io.IOUtils;
import org.apache.commons.io.input.BOMInputStream;
import org.apache.commons.lang3.StringUtils;
Expand Down Expand Up @@ -1461,9 +1462,15 @@ public Message pythonUpload(
}

// tar.gz包依赖检查
String errorMsg = FilesystemUtils.checkModuleFile(file, username);
if (StringUtils.isNotBlank(errorMsg)) {
return Message.error("部分依赖未加载，请检查并重新上传依赖包，依赖信息：" + errorMsg);
// 获取install_requires中的python模块
List<String> pythonModules = FilesystemUtils.getInstallRequestPythonModules(file);
String dependencies = "";
if (CollectionUtils.isNotEmpty(pythonModules)) {
dependencies = String.join(",", pythonModules);
String errorMsg = FilesystemUtils.checkModuleFile(pythonModules, username);
if (StringUtils.isNotBlank(errorMsg)) {
return Message.error("部分依赖未加载，请检查并重新上传依赖包，依赖信息：" + errorMsg);
}
}

// 定义目录路径
Expand Down Expand Up @@ -1499,11 +1506,12 @@ public Message pythonUpload(
OutputStream outputStream = null;
try {
String packageName = FilesystemUtils.findPackageName(file.getInputStream());
fileName = packageName + FsPath.CUR_DIR + "zip";
if (StringUtils.isBlank(packageName)) {
return Message.error("文件上传失败：PKG-INFO 文件不存在");
}
is = FilesystemUtils.getZipInputStreamByTarInputStream(file, packageName);
newPath = fsPath.getPath() + FsPath.SEPARATOR + fileName.replace(".tar.gz", ".zip");
newPath = fsPath.getPath() + FsPath.SEPARATOR + fileName;
FsPath fsPathNew = new FsPath(newPath);
outputStream = fileSystem.write(fsPathNew, true);
IOUtils.copy(is, outputStream);
Expand All @@ -1519,6 +1527,9 @@ public Message pythonUpload(
}
}
// 返回成功消息并包含文件地址
return Message.ok().data("filePath", newPath);
return Message.ok()
.data("filePath", newPath)
.data("dependencies", dependencies)
.data("fileName", fileName);
}
}
7 changes: 3 additions & 4 deletions ...is-pes-publicservice/src/main/java/org/apache/linkis/filesystem/util/FilesystemUtils.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -184,13 +184,12 @@ public static InputStream getZipInputStreamByTarInputStream(
* * 检查python环境中模块是否存在，使用程序调用脚本实现 脚本保存在admin文件夹中，脚本名为check_python_module.py
* 脚本执行存在返回ture，不存在返回false
*
* @param file 传入需要检查的模块列表
* @param pythonModules 传入需要检查的模块列表
* @param username
* @return 返回不存在的模块列表
*/
public static String checkModuleFile(MultipartFile file, String username) throws IOException {
// 获取install_requires中的python模块
List<String> pythonModules = getInstallRequestPythonModules(file);
public static String checkModuleFile(List<String> pythonModules, String username)
throws IOException {
StringJoiner joiner = new StringJoiner(",");
// 检查机器上pyhton环境中模块是否存在
List<String> notExistModules =
Expand Down
4 changes: 2 additions & 2 deletions ...nhancements/linkis-udf-service/src/main/java/org/apache/linkis/udf/api/UDFRestfulApi.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1290,10 +1290,10 @@ public Message pythonFileExist(
if (!fileNameWithoutExtension.matches("^[a-zA-Z][a-zA-Z0-9_-]{0,49}$")) {
return Message.error("只支持数字字母下划线，中划线，且以字母开头，长度最大50");
}

String fileNameWithoutVersion = fileNameWithoutExtension.split("-")[0];
// 封装PythonModuleInfo对象并查询数据库
PythonModuleInfo pythonModuleInfo = new PythonModuleInfo();
pythonModuleInfo.setName(fileNameWithoutExtension);
pythonModuleInfo.setName(fileNameWithoutVersion);
pythonModuleInfo.setCreateUser(userName);
PythonModuleInfo moduleInfo = pythonModuleInfoService.getByUserAndNameAndId(pythonModuleInfo);

Expand Down

0 comments on commit d997a03

Please sign in to comment.