From 0d94a5fe452f8ad8be7705c9ddbb715784a58f89 Mon Sep 17 00:00:00 2001 From: SbloodyS <460888207@qq.com> Date: Mon, 18 Nov 2024 15:15:10 +0800 Subject: [PATCH] feature 16801 --- .../dto/resources/visitor/ResourceTreeVisitor.java | 8 +------- .../common/constants/Constants.java | 2 -- .../plugin/storage/api/AbstractStorageOperator.java | 3 +-- .../plugin/storage/api/ResourceMetadata.java | 1 - .../plugin/storage/api/StorageEntity.java | 1 + .../plugin/storage/api/StorageOperator.java | 2 ++ .../plugin/storage/cos/CosStorageOperator.java | 1 + .../plugin/storage/gcs/GcsStorageOperator.java | 1 + .../plugin/storage/hdfs/HdfsStorageOperator.java | 2 +- .../storage/hdfs/LocalStorageOperatorTest.java | 13 ++++++------- .../plugin/storage/obs/ObsStorageOperator.java | 1 + .../plugin/storage/oss/OssStorageOperator.java | 1 + .../plugin/storage/s3/S3StorageOperator.java | 1 + 13 files changed, 17 insertions(+), 20 deletions(-) diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/visitor/ResourceTreeVisitor.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/visitor/ResourceTreeVisitor.java index c71d65a25974..f7f93e780db0 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/visitor/ResourceTreeVisitor.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/visitor/ResourceTreeVisitor.java @@ -20,7 +20,6 @@ import org.apache.dolphinscheduler.api.dto.resources.Directory; import org.apache.dolphinscheduler.api.dto.resources.FileLeaf; import org.apache.dolphinscheduler.api.dto.resources.ResourceComponent; -import org.apache.dolphinscheduler.common.constants.Constants; import org.apache.dolphinscheduler.plugin.storage.api.StorageEntity; import java.util.ArrayList; @@ -124,15 +123,10 @@ private static ResourceComponent getResourceComponent(StorageEntity resource) { tempResourceComponent = new FileLeaf(); } - String currentDir = ""; - if (resource.getFullName().contains(Constants.RESOURCE_TYPE_FILE)) { - currentDir = resource.getFullName().split(Constants.RESOURCE_TYPE_FILE)[1].substring(1); - } - tempResourceComponent.setName(resource.getFileName()); tempResourceComponent.setFullName(resource.getFullName()); tempResourceComponent.setType(resource.getType()); - tempResourceComponent.setCurrentDir(currentDir); + tempResourceComponent.setCurrentDir(resource.getRelativePath()); return tempResourceComponent; } diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/constants/Constants.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/constants/Constants.java index 254896a27cc0..cc01c45c1df1 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/constants/Constants.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/constants/Constants.java @@ -43,8 +43,6 @@ public final class Constants { public static final String FORMAT_S_S_COLON = "%s:%s"; public static final String FOLDER_SEPARATOR = "/"; - public static final String RESOURCE_TYPE_FILE = "resources"; - public static final String EMPTY_STRING = ""; /** diff --git a/dolphinscheduler-storage-plugin/dolphinscheduler-storage-api/src/main/java/org/apache/dolphinscheduler/plugin/storage/api/AbstractStorageOperator.java b/dolphinscheduler-storage-plugin/dolphinscheduler-storage-api/src/main/java/org/apache/dolphinscheduler/plugin/storage/api/AbstractStorageOperator.java index 885ccab09519..76a0f3096f24 100644 --- a/dolphinscheduler-storage-plugin/dolphinscheduler-storage-api/src/main/java/org/apache/dolphinscheduler/plugin/storage/api/AbstractStorageOperator.java +++ b/dolphinscheduler-storage-plugin/dolphinscheduler-storage-api/src/main/java/org/apache/dolphinscheduler/plugin/storage/api/AbstractStorageOperator.java @@ -17,7 +17,6 @@ package org.apache.dolphinscheduler.plugin.storage.api; -import org.apache.dolphinscheduler.common.constants.Constants; import org.apache.dolphinscheduler.common.utils.FileUtils; import org.apache.dolphinscheduler.common.utils.PropertyUtils; import org.apache.dolphinscheduler.plugin.storage.api.constants.StorageConstants; @@ -86,7 +85,7 @@ public String getStorageBaseDirectory(String tenantCode, ResourceType resourceTy String resourceBaseDirectory; switch (resourceType) { case FILE: - resourceBaseDirectory = FileUtils.concatFilePath(tenantBaseDirectory, Constants.RESOURCE_TYPE_FILE); + resourceBaseDirectory = FileUtils.concatFilePath(tenantBaseDirectory, StorageOperator.FILE_FOLDER_NAME); break; case ALL: resourceBaseDirectory = tenantBaseDirectory; diff --git a/dolphinscheduler-storage-plugin/dolphinscheduler-storage-api/src/main/java/org/apache/dolphinscheduler/plugin/storage/api/ResourceMetadata.java b/dolphinscheduler-storage-plugin/dolphinscheduler-storage-api/src/main/java/org/apache/dolphinscheduler/plugin/storage/api/ResourceMetadata.java index 6a2ee3dc6349..092ac7a6c6c3 100644 --- a/dolphinscheduler-storage-plugin/dolphinscheduler-storage-api/src/main/java/org/apache/dolphinscheduler/plugin/storage/api/ResourceMetadata.java +++ b/dolphinscheduler-storage-plugin/dolphinscheduler-storage-api/src/main/java/org/apache/dolphinscheduler/plugin/storage/api/ResourceMetadata.java @@ -38,5 +38,4 @@ public class ResourceMetadata { private String resourceRelativePath; private String resourceParentAbsolutePath; private boolean isDirectory; - } diff --git a/dolphinscheduler-storage-plugin/dolphinscheduler-storage-api/src/main/java/org/apache/dolphinscheduler/plugin/storage/api/StorageEntity.java b/dolphinscheduler-storage-plugin/dolphinscheduler-storage-api/src/main/java/org/apache/dolphinscheduler/plugin/storage/api/StorageEntity.java index ef9d58812999..e934321b4635 100644 --- a/dolphinscheduler-storage-plugin/dolphinscheduler-storage-api/src/main/java/org/apache/dolphinscheduler/plugin/storage/api/StorageEntity.java +++ b/dolphinscheduler-storage-plugin/dolphinscheduler-storage-api/src/main/java/org/apache/dolphinscheduler/plugin/storage/api/StorageEntity.java @@ -46,4 +46,5 @@ public class StorageEntity { private long size; private Date createTime; private Date updateTime; + private String relativePath; } diff --git a/dolphinscheduler-storage-plugin/dolphinscheduler-storage-api/src/main/java/org/apache/dolphinscheduler/plugin/storage/api/StorageOperator.java b/dolphinscheduler-storage-plugin/dolphinscheduler-storage-api/src/main/java/org/apache/dolphinscheduler/plugin/storage/api/StorageOperator.java index be49e4797c37..b7855120f1fc 100644 --- a/dolphinscheduler-storage-plugin/dolphinscheduler-storage-api/src/main/java/org/apache/dolphinscheduler/plugin/storage/api/StorageOperator.java +++ b/dolphinscheduler-storage-plugin/dolphinscheduler-storage-api/src/main/java/org/apache/dolphinscheduler/plugin/storage/api/StorageOperator.java @@ -24,6 +24,8 @@ public interface StorageOperator { + String FILE_FOLDER_NAME = "resources"; + ResourceMetadata getResourceMetaData(String resourceAbsolutePath); /** diff --git a/dolphinscheduler-storage-plugin/dolphinscheduler-storage-cos/src/main/java/org/apache/dolphinscheduler/plugin/storage/cos/CosStorageOperator.java b/dolphinscheduler-storage-plugin/dolphinscheduler-storage-cos/src/main/java/org/apache/dolphinscheduler/plugin/storage/cos/CosStorageOperator.java index a8eb50b33d23..e2f605ec917e 100644 --- a/dolphinscheduler-storage-plugin/dolphinscheduler-storage-cos/src/main/java/org/apache/dolphinscheduler/plugin/storage/cos/CosStorageOperator.java +++ b/dolphinscheduler-storage-plugin/dolphinscheduler-storage-cos/src/main/java/org/apache/dolphinscheduler/plugin/storage/cos/CosStorageOperator.java @@ -306,6 +306,7 @@ protected StorageEntity transformCOSObjectToStorageEntity(COSObject object) { .type(resourceMetaData.getResourceType()) .isDirectory(StringUtils.isEmpty(fileExtension)) .size(metadata.getContentLength()) + .relativePath(resourceMetaData.getResourceRelativePath()) .createTime(metadata.getLastModified()) .updateTime(metadata.getLastModified()) .build(); diff --git a/dolphinscheduler-storage-plugin/dolphinscheduler-storage-gcs/src/main/java/org/apache/dolphinscheduler/plugin/storage/gcs/GcsStorageOperator.java b/dolphinscheduler-storage-plugin/dolphinscheduler-storage-gcs/src/main/java/org/apache/dolphinscheduler/plugin/storage/gcs/GcsStorageOperator.java index 45f9c8ab6f52..fe85a402b813 100644 --- a/dolphinscheduler-storage-plugin/dolphinscheduler-storage-gcs/src/main/java/org/apache/dolphinscheduler/plugin/storage/gcs/GcsStorageOperator.java +++ b/dolphinscheduler-storage-plugin/dolphinscheduler-storage-gcs/src/main/java/org/apache/dolphinscheduler/plugin/storage/gcs/GcsStorageOperator.java @@ -273,6 +273,7 @@ private StorageEntity transformBlobToStorageEntity(Blob blob) { entity.setDirectory(resourceMetaData.isDirectory()); entity.setType(resourceMetaData.getResourceType()); entity.setSize(blob.getSize()); + entity.setRelativePath(resourceMetaData.getResourceRelativePath()); entity.setCreateTime(Date.from(blob.getCreateTimeOffsetDateTime().toInstant())); entity.setUpdateTime(Date.from(blob.getUpdateTimeOffsetDateTime().toInstant())); return entity; diff --git a/dolphinscheduler-storage-plugin/dolphinscheduler-storage-hdfs/src/main/java/org/apache/dolphinscheduler/plugin/storage/hdfs/HdfsStorageOperator.java b/dolphinscheduler-storage-plugin/dolphinscheduler-storage-hdfs/src/main/java/org/apache/dolphinscheduler/plugin/storage/hdfs/HdfsStorageOperator.java index 0d21b769a96d..3a1092077b90 100644 --- a/dolphinscheduler-storage-plugin/dolphinscheduler-storage-hdfs/src/main/java/org/apache/dolphinscheduler/plugin/storage/hdfs/HdfsStorageOperator.java +++ b/dolphinscheduler-storage-plugin/dolphinscheduler-storage-hdfs/src/main/java/org/apache/dolphinscheduler/plugin/storage/hdfs/HdfsStorageOperator.java @@ -268,7 +268,6 @@ private StorageEntity transformFileStatusToResourceMetadata(FileStatus fileStatu Path fileStatusPath = fileStatus.getPath(); String fileAbsolutePath = fileStatusPath.toString(); ResourceMetadata resourceMetaData = getResourceMetaData(fileAbsolutePath); - return StorageEntity.builder() .fileName(fileStatusPath.getName()) .fullName(fileAbsolutePath) @@ -276,6 +275,7 @@ private StorageEntity transformFileStatusToResourceMetadata(FileStatus fileStatu .type(resourceMetaData.getResourceType()) .isDirectory(fileStatus.isDirectory()) .size(fileStatus.getLen()) + .relativePath(resourceMetaData.getResourceRelativePath()) .createTime(new Date(fileStatus.getModificationTime())) .updateTime(new Date(fileStatus.getModificationTime())) .build(); diff --git a/dolphinscheduler-storage-plugin/dolphinscheduler-storage-hdfs/src/test/java/org/apache/dolphinscheduler/plugin/storage/hdfs/LocalStorageOperatorTest.java b/dolphinscheduler-storage-plugin/dolphinscheduler-storage-hdfs/src/test/java/org/apache/dolphinscheduler/plugin/storage/hdfs/LocalStorageOperatorTest.java index 5730f46dbd4b..899f17873a26 100644 --- a/dolphinscheduler-storage-plugin/dolphinscheduler-storage-hdfs/src/test/java/org/apache/dolphinscheduler/plugin/storage/hdfs/LocalStorageOperatorTest.java +++ b/dolphinscheduler-storage-plugin/dolphinscheduler-storage-hdfs/src/test/java/org/apache/dolphinscheduler/plugin/storage/hdfs/LocalStorageOperatorTest.java @@ -20,7 +20,6 @@ import static com.google.common.truth.Truth.assertThat; import static org.junit.jupiter.api.Assertions.assertThrows; -import org.apache.dolphinscheduler.common.constants.Constants; import org.apache.dolphinscheduler.common.utils.FileUtils; import org.apache.dolphinscheduler.plugin.storage.api.ResourceMetadata; import org.apache.dolphinscheduler.plugin.storage.api.StorageEntity; @@ -47,7 +46,7 @@ class LocalStorageOperatorTest { Paths.get(LocalStorageOperatorTest.class.getResource("/").getFile(), "localStorage").toString(); private static final String tenantCode = "default"; private static final String baseDir = - Paths.get(resourceBaseDir, tenantCode, StorageConstants.RESOURCE_TYPE_FILE).toString(); + Paths.get(resourceBaseDir, tenantCode, StorageOperator.FILE_FOLDER_NAME).toString(); @SneakyThrows @BeforeEach @@ -116,7 +115,7 @@ public void testGetStorageBaseDirectory_withTenant() { public void testGetStorageBaseDirectory_withTenant_withResourceTypeFile() { String storageBaseDirectory = storageOperator.getStorageBaseDirectory("default", ResourceType.FILE); assertThat(storageBaseDirectory) - .isEqualTo("file:" + Paths.get(resourceBaseDir, tenantCode, Constants.RESOURCE_TYPE_FILE)); + .isEqualTo("file:" + Paths.get(resourceBaseDir, tenantCode, StorageOperator.FILE_FOLDER_NAME)); } @Test @@ -143,21 +142,21 @@ public void testGetStorageBaseDirectory_withTenant_withEmptyResourceType() { public void testGetStorageFileAbsolutePath() { String fileAbsolutePath = storageOperator.getStorageFileAbsolutePath("default", "test.sh"); assertThat(fileAbsolutePath).isEqualTo( - "file:" + Paths.get(resourceBaseDir, tenantCode, Constants.RESOURCE_TYPE_FILE, "test.sh")); + "file:" + Paths.get(resourceBaseDir, tenantCode, StorageOperator.FILE_FOLDER_NAME, "test.sh")); } @SneakyThrows @Test public void testCreateStorageDir_notExists() { String testDirFileAbsolutePath = - "file:" + Paths.get(resourceBaseDir, "root", Constants.RESOURCE_TYPE_FILE, "testDir"); + "file:" + Paths.get(resourceBaseDir, "root", StorageOperator.FILE_FOLDER_NAME, "testDir"); try { storageOperator.createStorageDir(testDirFileAbsolutePath); StorageEntity storageEntity = storageOperator.getStorageEntity(testDirFileAbsolutePath); assertThat(storageEntity.getFullName()).isEqualTo(testDirFileAbsolutePath); assertThat(storageEntity.getFileName()).isEqualTo("testDir"); assertThat(storageEntity.getPfullName()) - .isEqualTo("file:" + Paths.get(resourceBaseDir, "root", Constants.RESOURCE_TYPE_FILE)); + .isEqualTo("file:" + Paths.get(resourceBaseDir, "root", StorageOperator.FILE_FOLDER_NAME)); assertThat(storageEntity.isDirectory()).isTrue(); assertThat(storageEntity.getType()).isEqualTo(ResourceType.FILE); } finally { @@ -169,7 +168,7 @@ public void testCreateStorageDir_notExists() { @Test public void testCreateStorageDir_exists() { String testDirFileAbsolutePath = - "file:" + Paths.get(resourceBaseDir, "default", Constants.RESOURCE_TYPE_FILE, "sqlDirectory"); + "file:" + Paths.get(resourceBaseDir, "default", StorageOperator.FILE_FOLDER_NAME, "sqlDirectory"); assertThrows(FileAlreadyExistsException.class, () -> storageOperator.createStorageDir(testDirFileAbsolutePath)); } diff --git a/dolphinscheduler-storage-plugin/dolphinscheduler-storage-obs/src/main/java/org/apache/dolphinscheduler/plugin/storage/obs/ObsStorageOperator.java b/dolphinscheduler-storage-plugin/dolphinscheduler-storage-obs/src/main/java/org/apache/dolphinscheduler/plugin/storage/obs/ObsStorageOperator.java index 1f32cb34afc3..129b9a88281f 100644 --- a/dolphinscheduler-storage-plugin/dolphinscheduler-storage-obs/src/main/java/org/apache/dolphinscheduler/plugin/storage/obs/ObsStorageOperator.java +++ b/dolphinscheduler-storage-plugin/dolphinscheduler-storage-obs/src/main/java/org/apache/dolphinscheduler/plugin/storage/obs/ObsStorageOperator.java @@ -260,6 +260,7 @@ protected StorageEntity transformObsObjectToStorageEntity(ObsObject object) { .type(resourceMetaData.getResourceType()) .isDirectory(StringUtils.isEmpty(fileExtension)) .size(metadata.getContentLength()) + .relativePath(resourceMetaData.getResourceRelativePath()) .createTime(metadata.getLastModified()) .updateTime(metadata.getLastModified()) .build(); diff --git a/dolphinscheduler-storage-plugin/dolphinscheduler-storage-oss/src/main/java/org/apache/dolphinscheduler/plugin/storage/oss/OssStorageOperator.java b/dolphinscheduler-storage-plugin/dolphinscheduler-storage-oss/src/main/java/org/apache/dolphinscheduler/plugin/storage/oss/OssStorageOperator.java index 4b142466ced8..0b78ce49b40a 100644 --- a/dolphinscheduler-storage-plugin/dolphinscheduler-storage-oss/src/main/java/org/apache/dolphinscheduler/plugin/storage/oss/OssStorageOperator.java +++ b/dolphinscheduler-storage-plugin/dolphinscheduler-storage-oss/src/main/java/org/apache/dolphinscheduler/plugin/storage/oss/OssStorageOperator.java @@ -331,6 +331,7 @@ protected StorageEntity transformOSSObjectToStorageEntity(OSSObject ossObject) { storageEntity.setType(resourceMetaData.getResourceType()); storageEntity.setDirectory(resourceMetaData.isDirectory()); storageEntity.setSize(ossObject.getObjectMetadata().getContentLength()); + storageEntity.setRelativePath(resourceMetaData.getResourceRelativePath()); storageEntity.setCreateTime(ossObject.getObjectMetadata().getLastModified()); storageEntity.setUpdateTime(ossObject.getObjectMetadata().getLastModified()); return storageEntity; diff --git a/dolphinscheduler-storage-plugin/dolphinscheduler-storage-s3/src/main/java/org/apache/dolphinscheduler/plugin/storage/s3/S3StorageOperator.java b/dolphinscheduler-storage-plugin/dolphinscheduler-storage-s3/src/main/java/org/apache/dolphinscheduler/plugin/storage/s3/S3StorageOperator.java index 2bd479de634a..d1836d7c9455 100644 --- a/dolphinscheduler-storage-plugin/dolphinscheduler-storage-s3/src/main/java/org/apache/dolphinscheduler/plugin/storage/s3/S3StorageOperator.java +++ b/dolphinscheduler-storage-plugin/dolphinscheduler-storage-s3/src/main/java/org/apache/dolphinscheduler/plugin/storage/s3/S3StorageOperator.java @@ -294,6 +294,7 @@ private StorageEntity transformS3ObjectToStorageEntity(S3Object object) { entity.setDirectory(resourceMetaData.isDirectory()); entity.setType(resourceMetaData.getResourceType()); entity.setSize(object.getObjectMetadata().getContentLength()); + entity.setRelativePath(resourceMetaData.getResourceRelativePath()); entity.setCreateTime(object.getObjectMetadata().getLastModified()); entity.setUpdateTime(object.getObjectMetadata().getLastModified()); return entity;