feature 16801

apache · Nov 18, 2024 · 0d94a5f · 0d94a5f
1 parent 6c2b7e0
commit 0d94a5f
Show file tree

Hide file tree

Showing 13 changed files with 17 additions and 20 deletions.
diff --git a/.../main/java/org/apache/dolphinscheduler/api/dto/resources/visitor/ResourceTreeVisitor.java b/.../main/java/org/apache/dolphinscheduler/api/dto/resources/visitor/ResourceTreeVisitor.java
@@ -20,7 +20,6 @@
 import org.apache.dolphinscheduler.api.dto.resources.Directory;
 import org.apache.dolphinscheduler.api.dto.resources.FileLeaf;
 import org.apache.dolphinscheduler.api.dto.resources.ResourceComponent;
-import org.apache.dolphinscheduler.common.constants.Constants;
 import org.apache.dolphinscheduler.plugin.storage.api.StorageEntity;
 
 import java.util.ArrayList;
@@ -124,15 +123,10 @@ private static ResourceComponent getResourceComponent(StorageEntity resource) {
             tempResourceComponent = new FileLeaf();
         }
 
-        String currentDir = "";
-        if (resource.getFullName().contains(Constants.RESOURCE_TYPE_FILE)) {
-            currentDir = resource.getFullName().split(Constants.RESOURCE_TYPE_FILE)[1].substring(1);
-        }
-
         tempResourceComponent.setName(resource.getFileName());
         tempResourceComponent.setFullName(resource.getFullName());
         tempResourceComponent.setType(resource.getType());
-        tempResourceComponent.setCurrentDir(currentDir);
+        tempResourceComponent.setCurrentDir(resource.getRelativePath());
         return tempResourceComponent;
     }
 

diff --git a/...cheduler-common/src/main/java/org/apache/dolphinscheduler/common/constants/Constants.java b/...cheduler-common/src/main/java/org/apache/dolphinscheduler/common/constants/Constants.java
@@ -43,8 +43,6 @@ public final class Constants {
     public static final String FORMAT_S_S_COLON = "%s:%s";
     public static final String FOLDER_SEPARATOR = "/";
 
-    public static final String RESOURCE_TYPE_FILE = "resources";
-
     public static final String EMPTY_STRING = "";
 
     /**

diff --git a/...src/main/java/org/apache/dolphinscheduler/plugin/storage/api/AbstractStorageOperator.java b/...src/main/java/org/apache/dolphinscheduler/plugin/storage/api/AbstractStorageOperator.java
@@ -17,7 +17,6 @@
 
 package org.apache.dolphinscheduler.plugin.storage.api;
 
-import org.apache.dolphinscheduler.common.constants.Constants;
 import org.apache.dolphinscheduler.common.utils.FileUtils;
 import org.apache.dolphinscheduler.common.utils.PropertyUtils;
 import org.apache.dolphinscheduler.plugin.storage.api.constants.StorageConstants;
@@ -86,7 +85,7 @@ public String getStorageBaseDirectory(String tenantCode, ResourceType resourceTy
         String resourceBaseDirectory;
         switch (resourceType) {
             case FILE:
-                resourceBaseDirectory = FileUtils.concatFilePath(tenantBaseDirectory, Constants.RESOURCE_TYPE_FILE);
+                resourceBaseDirectory = FileUtils.concatFilePath(tenantBaseDirectory, StorageOperator.FILE_FOLDER_NAME);
                 break;
             case ALL:
                 resourceBaseDirectory = tenantBaseDirectory;

diff --git a/...ge-api/src/main/java/org/apache/dolphinscheduler/plugin/storage/api/ResourceMetadata.java b/...ge-api/src/main/java/org/apache/dolphinscheduler/plugin/storage/api/ResourceMetadata.java
@@ -38,5 +38,4 @@ public class ResourceMetadata {
     private String resourceRelativePath;
     private String resourceParentAbsolutePath;
     private boolean isDirectory;
-
 }
diff --git a/...orage-api/src/main/java/org/apache/dolphinscheduler/plugin/storage/api/StorageEntity.java b/...orage-api/src/main/java/org/apache/dolphinscheduler/plugin/storage/api/StorageEntity.java
@@ -46,4 +46,5 @@ public class StorageEntity {
     private long size;
     private Date createTime;
     private Date updateTime;
+    private String relativePath;
 }
diff --git a/...age-api/src/main/java/org/apache/dolphinscheduler/plugin/storage/api/StorageOperator.java b/...age-api/src/main/java/org/apache/dolphinscheduler/plugin/storage/api/StorageOperator.java
@@ -24,6 +24,8 @@
 
 public interface StorageOperator {
 
+    String FILE_FOLDER_NAME = "resources";
+
     ResourceMetadata getResourceMetaData(String resourceAbsolutePath);
 
     /**

diff --git a/...-cos/src/main/java/org/apache/dolphinscheduler/plugin/storage/cos/CosStorageOperator.java b/...-cos/src/main/java/org/apache/dolphinscheduler/plugin/storage/cos/CosStorageOperator.java
@@ -306,6 +306,7 @@ protected StorageEntity transformCOSObjectToStorageEntity(COSObject object) {
                 .type(resourceMetaData.getResourceType())
                 .isDirectory(StringUtils.isEmpty(fileExtension))
                 .size(metadata.getContentLength())
+                .relativePath(resourceMetaData.getResourceRelativePath())
                 .createTime(metadata.getLastModified())
                 .updateTime(metadata.getLastModified())
                 .build();

diff --git a/...-gcs/src/main/java/org/apache/dolphinscheduler/plugin/storage/gcs/GcsStorageOperator.java b/...-gcs/src/main/java/org/apache/dolphinscheduler/plugin/storage/gcs/GcsStorageOperator.java
@@ -273,6 +273,7 @@ private StorageEntity transformBlobToStorageEntity(Blob blob) {
         entity.setDirectory(resourceMetaData.isDirectory());
         entity.setType(resourceMetaData.getResourceType());
         entity.setSize(blob.getSize());
+        entity.setRelativePath(resourceMetaData.getResourceRelativePath());
         entity.setCreateTime(Date.from(blob.getCreateTimeOffsetDateTime().toInstant()));
         entity.setUpdateTime(Date.from(blob.getUpdateTimeOffsetDateTime().toInstant()));
         return entity;

diff --git a/...fs/src/main/java/org/apache/dolphinscheduler/plugin/storage/hdfs/HdfsStorageOperator.java b/...fs/src/main/java/org/apache/dolphinscheduler/plugin/storage/hdfs/HdfsStorageOperator.java
@@ -268,14 +268,14 @@ private StorageEntity transformFileStatusToResourceMetadata(FileStatus fileStatu
         Path fileStatusPath = fileStatus.getPath();
         String fileAbsolutePath = fileStatusPath.toString();
         ResourceMetadata resourceMetaData = getResourceMetaData(fileAbsolutePath);
-
         return StorageEntity.builder()
                 .fileName(fileStatusPath.getName())
                 .fullName(fileAbsolutePath)
                 .pfullName(resourceMetaData.getResourceParentAbsolutePath())
                 .type(resourceMetaData.getResourceType())
                 .isDirectory(fileStatus.isDirectory())
                 .size(fileStatus.getLen())
+                .relativePath(resourceMetaData.getResourceRelativePath())
                 .createTime(new Date(fileStatus.getModificationTime()))
                 .updateTime(new Date(fileStatus.getModificationTime()))
                 .build();

diff --git a/...c/test/java/org/apache/dolphinscheduler/plugin/storage/hdfs/LocalStorageOperatorTest.java b/...c/test/java/org/apache/dolphinscheduler/plugin/storage/hdfs/LocalStorageOperatorTest.java
@@ -20,7 +20,6 @@
 import static com.google.common.truth.Truth.assertThat;
 import static org.junit.jupiter.api.Assertions.assertThrows;
 
-import org.apache.dolphinscheduler.common.constants.Constants;
 import org.apache.dolphinscheduler.common.utils.FileUtils;
 import org.apache.dolphinscheduler.plugin.storage.api.ResourceMetadata;
 import org.apache.dolphinscheduler.plugin.storage.api.StorageEntity;
@@ -47,7 +46,7 @@ class LocalStorageOperatorTest {
             Paths.get(LocalStorageOperatorTest.class.getResource("/").getFile(), "localStorage").toString();
     private static final String tenantCode = "default";
     private static final String baseDir =
-            Paths.get(resourceBaseDir, tenantCode, StorageConstants.RESOURCE_TYPE_FILE).toString();
+            Paths.get(resourceBaseDir, tenantCode, StorageOperator.FILE_FOLDER_NAME).toString();
 
     @SneakyThrows
     @BeforeEach
@@ -116,7 +115,7 @@ public void testGetStorageBaseDirectory_withTenant() {
     public void testGetStorageBaseDirectory_withTenant_withResourceTypeFile() {
         String storageBaseDirectory = storageOperator.getStorageBaseDirectory("default", ResourceType.FILE);
         assertThat(storageBaseDirectory)
-                .isEqualTo("file:" + Paths.get(resourceBaseDir, tenantCode, Constants.RESOURCE_TYPE_FILE));
+                .isEqualTo("file:" + Paths.get(resourceBaseDir, tenantCode, StorageOperator.FILE_FOLDER_NAME));
     }
 
     @Test
@@ -143,21 +142,21 @@ public void testGetStorageBaseDirectory_withTenant_withEmptyResourceType() {
     public void testGetStorageFileAbsolutePath() {
         String fileAbsolutePath = storageOperator.getStorageFileAbsolutePath("default", "test.sh");
         assertThat(fileAbsolutePath).isEqualTo(
-                "file:" + Paths.get(resourceBaseDir, tenantCode, Constants.RESOURCE_TYPE_FILE, "test.sh"));
+                "file:" + Paths.get(resourceBaseDir, tenantCode, StorageOperator.FILE_FOLDER_NAME, "test.sh"));
     }
 
     @SneakyThrows
     @Test
     public void testCreateStorageDir_notExists() {
         String testDirFileAbsolutePath =
-                "file:" + Paths.get(resourceBaseDir, "root", Constants.RESOURCE_TYPE_FILE, "testDir");
+                "file:" + Paths.get(resourceBaseDir, "root", StorageOperator.FILE_FOLDER_NAME, "testDir");
         try {
             storageOperator.createStorageDir(testDirFileAbsolutePath);
             StorageEntity storageEntity = storageOperator.getStorageEntity(testDirFileAbsolutePath);
             assertThat(storageEntity.getFullName()).isEqualTo(testDirFileAbsolutePath);
             assertThat(storageEntity.getFileName()).isEqualTo("testDir");
             assertThat(storageEntity.getPfullName())
-                    .isEqualTo("file:" + Paths.get(resourceBaseDir, "root", Constants.RESOURCE_TYPE_FILE));
+                    .isEqualTo("file:" + Paths.get(resourceBaseDir, "root", StorageOperator.FILE_FOLDER_NAME));
             assertThat(storageEntity.isDirectory()).isTrue();
             assertThat(storageEntity.getType()).isEqualTo(ResourceType.FILE);
         } finally {
@@ -169,7 +168,7 @@ public void testCreateStorageDir_notExists() {
     @Test
     public void testCreateStorageDir_exists() {
         String testDirFileAbsolutePath =
-                "file:" + Paths.get(resourceBaseDir, "default", Constants.RESOURCE_TYPE_FILE, "sqlDirectory");
+                "file:" + Paths.get(resourceBaseDir, "default", StorageOperator.FILE_FOLDER_NAME, "sqlDirectory");
         assertThrows(FileAlreadyExistsException.class, () -> storageOperator.createStorageDir(testDirFileAbsolutePath));
     }
 

diff --git a/...-obs/src/main/java/org/apache/dolphinscheduler/plugin/storage/obs/ObsStorageOperator.java b/...-obs/src/main/java/org/apache/dolphinscheduler/plugin/storage/obs/ObsStorageOperator.java
@@ -260,6 +260,7 @@ protected StorageEntity transformObsObjectToStorageEntity(ObsObject object) {
                 .type(resourceMetaData.getResourceType())
                 .isDirectory(StringUtils.isEmpty(fileExtension))
                 .size(metadata.getContentLength())
+                .relativePath(resourceMetaData.getResourceRelativePath())
                 .createTime(metadata.getLastModified())
                 .updateTime(metadata.getLastModified())
                 .build();

diff --git a/...-oss/src/main/java/org/apache/dolphinscheduler/plugin/storage/oss/OssStorageOperator.java b/...-oss/src/main/java/org/apache/dolphinscheduler/plugin/storage/oss/OssStorageOperator.java
@@ -331,6 +331,7 @@ protected StorageEntity transformOSSObjectToStorageEntity(OSSObject ossObject) {
         storageEntity.setType(resourceMetaData.getResourceType());
         storageEntity.setDirectory(resourceMetaData.isDirectory());
         storageEntity.setSize(ossObject.getObjectMetadata().getContentLength());
+        storageEntity.setRelativePath(resourceMetaData.getResourceRelativePath());
         storageEntity.setCreateTime(ossObject.getObjectMetadata().getLastModified());
         storageEntity.setUpdateTime(ossObject.getObjectMetadata().getLastModified());
         return storageEntity;

diff --git a/...age-s3/src/main/java/org/apache/dolphinscheduler/plugin/storage/s3/S3StorageOperator.java b/...age-s3/src/main/java/org/apache/dolphinscheduler/plugin/storage/s3/S3StorageOperator.java
@@ -294,6 +294,7 @@ private StorageEntity transformS3ObjectToStorageEntity(S3Object object) {
         entity.setDirectory(resourceMetaData.isDirectory());
         entity.setType(resourceMetaData.getResourceType());
         entity.setSize(object.getObjectMetadata().getContentLength());
+        entity.setRelativePath(resourceMetaData.getResourceRelativePath());
         entity.setCreateTime(object.getObjectMetadata().getLastModified());
         entity.setUpdateTime(object.getObjectMetadata().getLastModified());
         return entity;