From c68401b37732bca7415d72ceb230eb3d70f686fb Mon Sep 17 00:00:00 2001 From: yaoxuwan Date: Wed, 11 Sep 2024 15:22:19 +0800 Subject: [PATCH] =?UTF-8?q?feat:=20FileReference=E6=8A=BD=E5=8F=96?= =?UTF-8?q?=E5=88=B0common-metadata=20#2470?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat: FileReference抽取到common-metadata #2470 * feat: FileReference抽取到common-metadata #2470 --- .../archive/biz-archive/build.gradle.kts | 3 +- .../archive/core/compress/BDZipManager.kt | 10 +- .../archive/service/CompressServiceImpl.kt | 8 +- .../com/tencent/bkrepo/archive/BaseTest.kt | 2 +- .../archive/core/compress/BDZipManagerTest.kt | 9 +- .../common/artifact/manager/StorageManager.kt | 6 +- .../artifact/manager/StorageManagerTest.kt | 14 +-- .../metadata-api/build.gradle.kts | 4 + .../metadata}/pojo/file/FileReference.kt | 2 +- .../dao/{ => blocknode}/BlockNodeDao.kt | 2 +- .../dao/{ => blocknode}/RBlockNodeDao.kt | 2 +- .../metadata/dao/file}/FileReferenceDao.kt | 7 +- .../metadata/dao/file/RFileReferenceDao.kt | 45 +++++++ .../common/metadata}/model/TFileReference.kt | 2 +- .../impl/AbstractBlockNodeService.kt | 10 +- .../impl/DefaultBlockNodeServiceImpl.kt | 11 +- .../impl/RAbstractBlockNodeService.kt | 10 +- .../service/file/FileReferenceService.kt | 21 +--- .../service/file/RFileReferenceService.kt | 81 ++++++++++++ .../file/impl/FileReferenceServiceImpl.kt | 71 ++--------- .../file/impl/RFileReferenceServiceImpl.kt | 115 ++++++++++++++++++ .../metadata/util/FileReferenceQueryHelper.kt | 15 +++ .../reactive/dao/AbstractMongoReactiveDao.kt | 16 +++ .../mongo/reactive/dao/MongoReactiveDao.kt | 10 ++ .../bkrepo/fs/server/api/RRepositoryClient.kt | 6 - .../fs/server/service/BlockNodeServiceImpl.kt | 12 +- .../bkrepo/fs/service/BlockNodeServiceTest.kt | 19 +-- .../batch/task/archive/IdleNodeArchiveJob.kt | 12 +- .../task/clean/DeletedBlockNodeCleanupJob.kt | 6 +- .../job/batch/task/other/NodeCopyJob.kt | 8 +- .../batch/task/storage/StorageReconcileJob.kt | 8 +- .../job/migrate/executor/BaseTaskExecutor.kt | 10 +- .../job/migrate/executor/CorrectExecutor.kt | 6 +- .../job/migrate/executor/FinishExecutor.kt | 6 +- .../job/migrate/executor/MigrateExecutor.kt | 6 +- .../executor/MigrateFailedNodeExecutor.kt | 6 +- .../strategy/FileNotFoundAutoFixStrategy.kt | 6 +- .../tencent/bkrepo/job/batch/JobBaseTest.kt | 2 +- .../task/storage/StorageReconcileJobTest.kt | 12 +- .../migrate/executor/CorrectExecutorTest.kt | 3 +- .../job/migrate/executor/ExecutorBaseTest.kt | 11 +- .../FileNotFoundAutoFixStrategyTest.kt | 13 +- .../repository/api/FileReferenceClient.kt | 1 + .../bkrepo/repository/api/RepositoryClient.kt | 2 - .../api/StorageCredentialsClient.kt | 2 +- .../proxy/ProxyFileReferenceController.kt | 2 +- .../service/FileReferenceController.kt | 2 +- .../service/StorageCredentialsController.kt | 2 + .../user/UserFileReferenceController.kt | 4 +- .../repository/job/DeletedNodeCleanupJob.kt | 6 +- .../service/blocknode/BlockNodeServiceImpl.kt | 12 +- .../service/node/impl/NodeBaseService.kt | 33 ++++- .../service/node/impl/NodeServiceImpl.kt | 2 +- .../center/CommitEdgeCenterNodeServiceImpl.kt | 2 +- .../node/impl/edge/EdgeNodeBaseService.kt | 2 +- .../node/impl/edge/EdgeNodeServiceImpl.kt | 2 +- .../service/FileReferenceServiceTest.kt | 4 +- .../repository/service/MetadataServiceTest.kt | 2 +- .../NodeQueryWithoutShardingKeyTest.kt | 2 +- .../service/NodeSearchServiceTest.kt | 2 +- .../repository/service/NodeServiceTest.kt | 2 +- .../repository/service/ServiceBaseTest.kt | 4 +- .../service/StorageCredentialServiceTest.kt | 2 +- .../service/node/NodeRestoreSupportTest.kt | 2 +- 64 files changed, 473 insertions(+), 249 deletions(-) rename src/backend/{repository/api-repository/src/main/kotlin/com/tencent/bkrepo/repository => common/common-metadata/metadata-api/src/main/kotlin/com/tencent/bkrepo/common/metadata}/pojo/file/FileReference.kt (97%) rename src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/dao/{ => blocknode}/BlockNodeDao.kt (97%) rename src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/dao/{ => blocknode}/RBlockNodeDao.kt (97%) rename src/backend/{repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/dao => common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/dao/file}/FileReferenceDao.kt (86%) create mode 100644 src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/dao/file/RFileReferenceDao.kt rename src/backend/{repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository => common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata}/model/TFileReference.kt (97%) rename src/backend/{repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository => common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata}/service/file/FileReferenceService.kt (74%) create mode 100644 src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/file/RFileReferenceService.kt rename src/backend/{repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository => common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata}/service/file/impl/FileReferenceServiceImpl.kt (61%) create mode 100644 src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/file/impl/RFileReferenceServiceImpl.kt create mode 100644 src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/util/FileReferenceQueryHelper.kt diff --git a/src/backend/archive/biz-archive/build.gradle.kts b/src/backend/archive/biz-archive/build.gradle.kts index 356a9156ec..ad7290650a 100644 --- a/src/backend/archive/biz-archive/build.gradle.kts +++ b/src/backend/archive/biz-archive/build.gradle.kts @@ -6,10 +6,11 @@ dependencies { api(project(":common:common-service")) api(project(":common:common-mongo")) api(project(":common:common-mongo-reactive")) + api(project(":common:common-metadata:metadata-service")) implementation("io.micrometer:micrometer-registry-prometheus") implementation("org.apache.tika:tika-core") testImplementation("org.springframework.boot:spring-boot-starter-test") testImplementation("de.flapdoodle.embed:de.flapdoodle.embed.mongo") testImplementation("org.mockito.kotlin:mockito-kotlin") testImplementation("io.mockk:mockk") -} \ No newline at end of file +} diff --git a/src/backend/archive/biz-archive/src/main/kotlin/com/tencent/bkrepo/archive/core/compress/BDZipManager.kt b/src/backend/archive/biz-archive/src/main/kotlin/com/tencent/bkrepo/archive/core/compress/BDZipManager.kt index 237bc9e393..0cebe9fa72 100644 --- a/src/backend/archive/biz-archive/src/main/kotlin/com/tencent/bkrepo/archive/core/compress/BDZipManager.kt +++ b/src/backend/archive/biz-archive/src/main/kotlin/com/tencent/bkrepo/archive/core/compress/BDZipManager.kt @@ -3,12 +3,12 @@ package com.tencent.bkrepo.archive.core.compress import com.google.common.util.concurrent.ThreadFactoryBuilder import com.tencent.bkrepo.archive.CompressStatus import com.tencent.bkrepo.archive.config.ArchiveProperties +import com.tencent.bkrepo.archive.core.TaskResult import com.tencent.bkrepo.archive.core.provider.FileProviderFactory import com.tencent.bkrepo.archive.core.provider.FileTask +import com.tencent.bkrepo.archive.core.provider.PriorityFileProvider import com.tencent.bkrepo.archive.event.StorageFileCompressedEvent import com.tencent.bkrepo.archive.event.StorageFileUncompressedEvent -import com.tencent.bkrepo.archive.core.provider.PriorityFileProvider -import com.tencent.bkrepo.archive.core.TaskResult import com.tencent.bkrepo.archive.model.TCompressFile import com.tencent.bkrepo.archive.repository.CompressFileDao import com.tencent.bkrepo.archive.repository.CompressFileRepository @@ -17,11 +17,11 @@ import com.tencent.bkrepo.archive.utils.ArchiveUtils import com.tencent.bkrepo.common.artifact.api.toArtifactFile import com.tencent.bkrepo.common.artifact.stream.Range import com.tencent.bkrepo.common.bksync.transfer.exception.TooLowerReuseRateException +import com.tencent.bkrepo.common.metadata.service.file.FileReferenceService import com.tencent.bkrepo.common.service.util.SpringContextUtils import com.tencent.bkrepo.common.storage.core.StorageService import com.tencent.bkrepo.common.storage.monitor.Throughput import com.tencent.bkrepo.common.storage.util.toPath -import com.tencent.bkrepo.repository.api.FileReferenceClient import org.slf4j.LoggerFactory import org.springframework.beans.factory.annotation.Autowired import org.springframework.context.annotation.Lazy @@ -50,7 +50,7 @@ class BDZipManager( @Autowired @Lazy - private lateinit var fileReferenceClient: FileReferenceClient + private lateinit var fileReferenceService: FileReferenceService @Autowired @Lazy @@ -190,7 +190,7 @@ class BDZipManager( if (it !is TooLowerReuseRateException) { logger.error("Failed to compress file [$sha256].", it) } - fileReferenceClient.decrement(baseSha256, storageCredentialsKey) + fileReferenceService.decrement(baseSha256, storageCredentialsKey) file.status = CompressStatus.COMPRESS_FAILED file.lastModifiedDate = LocalDateTime.now() compressFileRepository.save(file) diff --git a/src/backend/archive/biz-archive/src/main/kotlin/com/tencent/bkrepo/archive/service/CompressServiceImpl.kt b/src/backend/archive/biz-archive/src/main/kotlin/com/tencent/bkrepo/archive/service/CompressServiceImpl.kt index 103644f411..8f8bbbf1d2 100644 --- a/src/backend/archive/biz-archive/src/main/kotlin/com/tencent/bkrepo/archive/service/CompressServiceImpl.kt +++ b/src/backend/archive/biz-archive/src/main/kotlin/com/tencent/bkrepo/archive/service/CompressServiceImpl.kt @@ -15,9 +15,9 @@ import com.tencent.bkrepo.archive.request.UncompressFileRequest import com.tencent.bkrepo.archive.request.UpdateCompressFileStatusRequest import com.tencent.bkrepo.archive.utils.ArchiveUtils import com.tencent.bkrepo.common.api.exception.ErrorCodeException +import com.tencent.bkrepo.common.metadata.service.file.FileReferenceService import com.tencent.bkrepo.common.service.util.SpringContextUtils import com.tencent.bkrepo.common.storage.core.StorageService -import com.tencent.bkrepo.repository.api.FileReferenceClient import org.slf4j.LoggerFactory import org.springframework.stereotype.Service import java.time.LocalDateTime @@ -29,7 +29,7 @@ import java.time.LocalDateTime class CompressServiceImpl( private val compressFileRepository: CompressFileRepository, private val storageService: StorageService, - private val fileReferenceClient: FileReferenceClient, + private val fileReferenceService: FileReferenceService, ) : CompressService { override fun compress(request: CompressFileRequest) { @@ -99,7 +99,7 @@ class CompressServiceImpl( status = CompressStatus.CREATED, ) compressFileRepository.save(compressFile) - fileReferenceClient.increment(baseSha256, storageCredentialsKey) + fileReferenceService.increment(baseSha256, storageCredentialsKey) SpringContextUtils.publishEvent(FileEntityEvent(sha256, compressFile)) logger.info("Compress file [$sha256] on $storageCredentialsKey.") } @@ -133,7 +133,7 @@ class CompressServiceImpl( storageService.deleteCompressed(sha256, storageCredentials) } // 压缩失败的已经解除了base sha256的引用 - fileReferenceClient.decrement(file.baseSha256, storageCredentialsKey) + fileReferenceService.decrement(file.baseSha256, storageCredentialsKey) } /* * 解压是小概率事件，所以这里链长度我们就不减少，这样带来的问题是， diff --git a/src/backend/archive/biz-archive/src/test/kotlin/com/tencent/bkrepo/archive/BaseTest.kt b/src/backend/archive/biz-archive/src/test/kotlin/com/tencent/bkrepo/archive/BaseTest.kt index dda9025035..8f244837cf 100644 --- a/src/backend/archive/biz-archive/src/test/kotlin/com/tencent/bkrepo/archive/BaseTest.kt +++ b/src/backend/archive/biz-archive/src/test/kotlin/com/tencent/bkrepo/archive/BaseTest.kt @@ -14,7 +14,7 @@ import org.springframework.cloud.sleuth.otel.bridge.OtelTracer import org.springframework.context.annotation.ComponentScan import org.springframework.test.context.TestPropertySource -@ComponentScan("com.tencent.bkrepo.archive") +@ComponentScan(value = ["com.tencent.bkrepo.archive", "com.tencent.bkrepo.common.metadata"]) @SpringBootConfiguration @EnableAutoConfiguration @TestPropertySource(locations = ["classpath:bootstrap-ut.properties"]) diff --git a/src/backend/archive/biz-archive/src/test/kotlin/com/tencent/bkrepo/archive/core/compress/BDZipManagerTest.kt b/src/backend/archive/biz-archive/src/test/kotlin/com/tencent/bkrepo/archive/core/compress/BDZipManagerTest.kt index f59412e466..21f4067ce5 100644 --- a/src/backend/archive/biz-archive/src/test/kotlin/com/tencent/bkrepo/archive/core/compress/BDZipManagerTest.kt +++ b/src/backend/archive/biz-archive/src/test/kotlin/com/tencent/bkrepo/archive/core/compress/BDZipManagerTest.kt @@ -10,10 +10,9 @@ import com.tencent.bkrepo.common.artifact.api.FileSystemArtifactFile import com.tencent.bkrepo.common.artifact.hash.sha256 import com.tencent.bkrepo.common.artifact.stream.Range import com.tencent.bkrepo.common.bksync.file.BkSyncDeltaSource.Companion.toBkSyncDeltaSource -import com.tencent.bkrepo.common.service.util.ResponseBuilder +import com.tencent.bkrepo.common.metadata.service.file.FileReferenceService import com.tencent.bkrepo.common.storage.StorageAutoConfiguration import com.tencent.bkrepo.common.storage.core.StorageService -import com.tencent.bkrepo.repository.api.FileReferenceClient import com.tencent.bkrepo.repository.api.RepositoryClient import com.tencent.bkrepo.repository.api.StorageCredentialsClient import org.junit.jupiter.api.AfterEach @@ -44,7 +43,7 @@ class BDZipManagerTest @Autowired constructor( ) : BaseTest() { @MockBean - lateinit var fileReferenceClient: FileReferenceClient + lateinit var fileReferenceService: FileReferenceService @MockBean lateinit var storageCredentialsClient: StorageCredentialsClient @@ -84,10 +83,10 @@ class BDZipManagerTest @Autowired constructor( @Test fun compressFailedTest() { var decrement = 0 - Mockito.`when`(fileReferenceClient.decrement(ArgumentMatchers.anyString(), ArgumentMatchers.isNull())).then { + Mockito.`when`(fileReferenceService.decrement(ArgumentMatchers.anyString(), ArgumentMatchers.isNull())).then { println("decrement file reference") decrement++ - ResponseBuilder.success(true) + true } val data1 = Random.nextBytes(Random.nextInt(1024, 1 shl 20)) val data2 = data1.copyOfRange(Random.nextInt(1, 10), data1.size) diff --git a/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/manager/StorageManager.kt b/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/manager/StorageManager.kt index 3dea1c22a4..312be98fca 100644 --- a/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/manager/StorageManager.kt +++ b/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/manager/StorageManager.kt @@ -35,12 +35,12 @@ import com.tencent.bkrepo.common.artifact.stream.ArtifactInputStream import com.tencent.bkrepo.common.artifact.stream.EmptyInputStream import com.tencent.bkrepo.common.artifact.stream.Range import com.tencent.bkrepo.common.artifact.util.http.HttpRangeUtils.resolveRange +import com.tencent.bkrepo.common.metadata.service.file.FileReferenceService import com.tencent.bkrepo.common.security.util.SecurityUtils import com.tencent.bkrepo.common.service.util.HttpContextHolder.getRequestOrNull import com.tencent.bkrepo.common.storage.core.StorageService import com.tencent.bkrepo.common.storage.credentials.StorageCredentials import com.tencent.bkrepo.common.storage.innercos.http.HttpMethod -import com.tencent.bkrepo.repository.api.FileReferenceClient import com.tencent.bkrepo.repository.api.NodeClient import com.tencent.bkrepo.repository.pojo.node.NodeDetail import com.tencent.bkrepo.repository.pojo.node.NodeInfo @@ -66,7 +66,7 @@ import org.slf4j.LoggerFactory class StorageManager( private val storageService: StorageService, private val nodeClient: NodeClient, - private val fileReferenceClient: FileReferenceClient, + private val fileReferenceService: FileReferenceService, private val nodeResourceFactory: NodeResourceFactory, private val pluginManager: PluginManager, ) { @@ -88,7 +88,7 @@ class StorageManager( try { // 当createNode调用超时，实际node和引用创建成功时不会做任何改变 // 当文件创建成功，但是node创建失败时，则创建一个计数为0的fileReference用于清理任务清理垃圾文件 - fileReferenceClient.increment(request.sha256!!, storageCredentials?.key, 0L) + fileReferenceService.increment(request.sha256!!, storageCredentials?.key, 0L) } catch (exception: Exception) { // 创建引用失败后会通过定时任务StorageReconcileJob清理垃圾文件 logger.error("Failed to create ref for new created file[${request.sha256}]", exception) diff --git a/src/backend/common/common-artifact/artifact-service/src/test/kotlin/com/tencent/bkrepo/common/artifact/manager/StorageManagerTest.kt b/src/backend/common/common-artifact/artifact-service/src/test/kotlin/com/tencent/bkrepo/common/artifact/manager/StorageManagerTest.kt index 8dde16d32f..a6b75c7239 100644 --- a/src/backend/common/common-artifact/artifact-service/src/test/kotlin/com/tencent/bkrepo/common/artifact/manager/StorageManagerTest.kt +++ b/src/backend/common/common-artifact/artifact-service/src/test/kotlin/com/tencent/bkrepo/common/artifact/manager/StorageManagerTest.kt @@ -8,8 +8,8 @@ import com.tencent.bkrepo.common.artifact.util.Constant.UT_PROJECT_ID import com.tencent.bkrepo.common.artifact.util.Constant.UT_REPO_NAME import com.tencent.bkrepo.common.artifact.util.Constant.UT_SHA256 import com.tencent.bkrepo.common.artifact.util.Constant.UT_USER +import com.tencent.bkrepo.common.metadata.service.file.FileReferenceService import com.tencent.bkrepo.common.storage.core.StorageService -import com.tencent.bkrepo.repository.api.FileReferenceClient import com.tencent.bkrepo.repository.api.NodeClient import com.tencent.bkrepo.repository.pojo.node.NodeDetail import com.tencent.bkrepo.repository.pojo.node.NodeInfo @@ -50,7 +50,7 @@ class StorageManagerTest @Autowired constructor( private val storageManager: StorageManager, ) { @MockBean - private lateinit var fileReferenceClient: FileReferenceClient + private lateinit var fileReferenceService: FileReferenceService @MockBean private lateinit var nodeClient: NodeClient @@ -70,15 +70,15 @@ class StorageManagerTest @Autowired constructor( .thenReturn(1) whenever(nodeClient.createNode(any())) .thenReturn(Response(code = 0, data = buildNodeDetail(UT_SHA256))) - whenever(fileReferenceClient.increment(anyString(), anyOrNull(), any())) - .thenReturn(Response(code = 0, data = true)) + whenever(fileReferenceService.increment(anyString(), anyOrNull(), any())) + .thenReturn(true) } @Test fun testStoreSuccess() { store() verify(nodeClient, times(1)).createNode(any()) - verify(fileReferenceClient, times(0)).increment(anyString(), anyOrNull(), any()) + verify(fileReferenceService, times(0)).increment(anyString(), anyOrNull(), any()) } @Test @@ -94,7 +94,7 @@ class StorageManagerTest @Autowired constructor( // store failed assertThrows { store() } verify(nodeClient, times(0)).createNode(any()) - verify(fileReferenceClient, times(0)).increment(anyString(), anyOrNull(), any()) + verify(fileReferenceService, times(0)).increment(anyString(), anyOrNull(), any()) // reset mock field.set(storageManager, oldStorageService) @@ -108,7 +108,7 @@ class StorageManagerTest @Autowired constructor( // store failed assertThrows { store() } verify(nodeClient, times(1)).createNode(any()) - verify(fileReferenceClient, times(1)).increment(anyString(), anyOrNull(), any()) + verify(fileReferenceService, times(1)).increment(anyString(), anyOrNull(), any()) } private fun store(): String { diff --git a/src/backend/common/common-metadata/metadata-api/build.gradle.kts b/src/backend/common/common-metadata/metadata-api/build.gradle.kts index 6971910f79..148d433957 100644 --- a/src/backend/common/common-metadata/metadata-api/build.gradle.kts +++ b/src/backend/common/common-metadata/metadata-api/build.gradle.kts @@ -28,3 +28,7 @@ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ + +dependencies { + api(project(":common:common-api")) +} diff --git a/src/backend/repository/api-repository/src/main/kotlin/com/tencent/bkrepo/repository/pojo/file/FileReference.kt b/src/backend/common/common-metadata/metadata-api/src/main/kotlin/com/tencent/bkrepo/common/metadata/pojo/file/FileReference.kt similarity index 97% rename from src/backend/repository/api-repository/src/main/kotlin/com/tencent/bkrepo/repository/pojo/file/FileReference.kt rename to src/backend/common/common-metadata/metadata-api/src/main/kotlin/com/tencent/bkrepo/common/metadata/pojo/file/FileReference.kt index b52698b8d7..e60a534134 100644 --- a/src/backend/repository/api-repository/src/main/kotlin/com/tencent/bkrepo/repository/pojo/file/FileReference.kt +++ b/src/backend/common/common-metadata/metadata-api/src/main/kotlin/com/tencent/bkrepo/common/metadata/pojo/file/FileReference.kt @@ -25,7 +25,7 @@ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ -package com.tencent.bkrepo.repository.pojo.file +package com.tencent.bkrepo.common.metadata.pojo.file import io.swagger.annotations.ApiModel import io.swagger.annotations.ApiModelProperty diff --git a/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/dao/BlockNodeDao.kt b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/dao/blocknode/BlockNodeDao.kt similarity index 97% rename from src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/dao/BlockNodeDao.kt rename to src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/dao/blocknode/BlockNodeDao.kt index 3e93f78bab..3dc8412b1b 100644 --- a/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/dao/BlockNodeDao.kt +++ b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/dao/blocknode/BlockNodeDao.kt @@ -25,7 +25,7 @@ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ -package com.tencent.bkrepo.common.metadata.dao +package com.tencent.bkrepo.common.metadata.dao.blocknode import com.tencent.bkrepo.common.metadata.condition.SyncCondition import com.tencent.bkrepo.common.metadata.model.TBlockNode diff --git a/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/dao/RBlockNodeDao.kt b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/dao/blocknode/RBlockNodeDao.kt similarity index 97% rename from src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/dao/RBlockNodeDao.kt rename to src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/dao/blocknode/RBlockNodeDao.kt index 7ccb332148..f48c38fdb2 100644 --- a/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/dao/RBlockNodeDao.kt +++ b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/dao/blocknode/RBlockNodeDao.kt @@ -25,7 +25,7 @@ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ -package com.tencent.bkrepo.common.metadata.dao +package com.tencent.bkrepo.common.metadata.dao.blocknode import com.tencent.bkrepo.common.metadata.condition.ReactiveCondition import com.tencent.bkrepo.common.metadata.model.TBlockNode diff --git a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/dao/FileReferenceDao.kt b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/dao/file/FileReferenceDao.kt similarity index 86% rename from src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/dao/FileReferenceDao.kt rename to src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/dao/file/FileReferenceDao.kt index 84fe13b158..9124344dc8 100644 --- a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/dao/FileReferenceDao.kt +++ b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/dao/file/FileReferenceDao.kt @@ -29,14 +29,17 @@ * SOFTWARE. */ -package com.tencent.bkrepo.repository.dao +package com.tencent.bkrepo.common.metadata.dao.file +import com.tencent.bkrepo.common.metadata.condition.SyncCondition import com.tencent.bkrepo.common.mongo.dao.sharding.HashShardingMongoDao -import com.tencent.bkrepo.repository.model.TFileReference +import com.tencent.bkrepo.common.metadata.model.TFileReference +import org.springframework.context.annotation.Conditional import org.springframework.stereotype.Repository /** * 文件摘要引用 Dao */ @Repository +@Conditional(SyncCondition::class) class FileReferenceDao : HashShardingMongoDao() diff --git a/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/dao/file/RFileReferenceDao.kt b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/dao/file/RFileReferenceDao.kt new file mode 100644 index 0000000000..47fe6a9d10 --- /dev/null +++ b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/dao/file/RFileReferenceDao.kt @@ -0,0 +1,45 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package com.tencent.bkrepo.common.metadata.dao.file + +import com.tencent.bkrepo.common.metadata.condition.ReactiveCondition +import com.tencent.bkrepo.common.metadata.model.TFileReference +import com.tencent.bkrepo.common.mongo.reactive.dao.ShardingMongoReactiveDao +import org.springframework.context.annotation.Conditional +import org.springframework.stereotype.Repository + +/** + * 文件摘要引用 Dao + */ +@Repository +@Conditional(ReactiveCondition::class) +class RFileReferenceDao : ShardingMongoReactiveDao() diff --git a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/model/TFileReference.kt b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/model/TFileReference.kt similarity index 97% rename from src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/model/TFileReference.kt rename to src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/model/TFileReference.kt index cc9750d100..84e7f56b2e 100644 --- a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/model/TFileReference.kt +++ b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/model/TFileReference.kt @@ -29,7 +29,7 @@ * SOFTWARE. */ -package com.tencent.bkrepo.repository.model +package com.tencent.bkrepo.common.metadata.model import com.tencent.bkrepo.common.api.mongo.ShardingDocument import com.tencent.bkrepo.common.api.mongo.ShardingKey diff --git a/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/blocknode/impl/AbstractBlockNodeService.kt b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/blocknode/impl/AbstractBlockNodeService.kt index edb6cfeb7a..aded3986f7 100644 --- a/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/blocknode/impl/AbstractBlockNodeService.kt +++ b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/blocknode/impl/AbstractBlockNodeService.kt @@ -30,9 +30,10 @@ package com.tencent.bkrepo.common.metadata.service.blocknode.impl import com.tencent.bkrepo.common.artifact.stream.Range import com.tencent.bkrepo.common.metadata.constant.FAKE_SHA256 import com.tencent.bkrepo.common.metadata.constant.ID -import com.tencent.bkrepo.common.metadata.dao.BlockNodeDao +import com.tencent.bkrepo.common.metadata.dao.blocknode.BlockNodeDao import com.tencent.bkrepo.common.metadata.model.TBlockNode import com.tencent.bkrepo.common.metadata.service.blocknode.BlockNodeService +import com.tencent.bkrepo.common.metadata.service.file.FileReferenceService import com.tencent.bkrepo.common.metadata.util.BlockNodeQueryHelper import com.tencent.bkrepo.common.storage.credentials.StorageCredentials import com.tencent.bkrepo.common.storage.pojo.RegionResource @@ -46,13 +47,14 @@ import org.springframework.data.mongodb.core.query.isEqualTo import java.time.LocalDateTime abstract class AbstractBlockNodeService( - private val blockNodeDao: BlockNodeDao + private val blockNodeDao: BlockNodeDao, + private val fileReferenceService: FileReferenceService ) : BlockNodeService { override fun createBlock(blockNode: TBlockNode, storageCredentials: StorageCredentials?): TBlockNode { with(blockNode) { val bn = blockNodeDao.save(blockNode) - incFileRef(bn.sha256, storageCredentials?.key) + fileReferenceService.increment(bn.sha256, storageCredentials?.key) logger.info("Create block node[$projectId/$repoName$nodeFullPath-$startPos] ,sha256[$sha256] success.") return bn } @@ -131,8 +133,6 @@ abstract class AbstractBlockNodeService( } } - abstract fun incFileRef(sha256: String, credentialsKey: String?) - abstract fun getNodeDetail(projectId: String, repoName: String, fullPath: String): NodeDetail companion object { diff --git a/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/blocknode/impl/DefaultBlockNodeServiceImpl.kt b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/blocknode/impl/DefaultBlockNodeServiceImpl.kt index 688bd865b4..402590b0ad 100644 --- a/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/blocknode/impl/DefaultBlockNodeServiceImpl.kt +++ b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/blocknode/impl/DefaultBlockNodeServiceImpl.kt @@ -2,8 +2,8 @@ package com.tencent.bkrepo.common.metadata.service.blocknode.impl import com.tencent.bkrepo.common.artifact.exception.NodeNotFoundException import com.tencent.bkrepo.common.metadata.condition.SyncCondition -import com.tencent.bkrepo.common.metadata.dao.BlockNodeDao -import com.tencent.bkrepo.repository.api.FileReferenceClient +import com.tencent.bkrepo.common.metadata.dao.blocknode.BlockNodeDao +import com.tencent.bkrepo.common.metadata.service.file.FileReferenceService import com.tencent.bkrepo.repository.api.NodeClient import com.tencent.bkrepo.repository.pojo.node.NodeDetail import org.springframework.context.annotation.Conditional @@ -13,12 +13,9 @@ import org.springframework.stereotype.Service @Conditional(SyncCondition::class) class DefaultBlockNodeServiceImpl( blockNodeDao: BlockNodeDao, - private val fileReferenceClient: FileReferenceClient, + fileReferenceService: FileReferenceService, private val nodeClient: NodeClient -) : AbstractBlockNodeService(blockNodeDao) { - override fun incFileRef(sha256: String, credentialsKey: String?) { - fileReferenceClient.increment(sha256, credentialsKey) - } +) : AbstractBlockNodeService(blockNodeDao, fileReferenceService) { override fun getNodeDetail(projectId: String, repoName: String, fullPath: String): NodeDetail { return nodeClient.getNodeDetail(projectId, repoName, fullPath).data ?: throw NodeNotFoundException(fullPath) diff --git a/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/blocknode/impl/RAbstractBlockNodeService.kt b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/blocknode/impl/RAbstractBlockNodeService.kt index 4e48e22f5b..c844f21911 100644 --- a/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/blocknode/impl/RAbstractBlockNodeService.kt +++ b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/blocknode/impl/RAbstractBlockNodeService.kt @@ -30,9 +30,10 @@ package com.tencent.bkrepo.common.metadata.service.blocknode.impl import com.tencent.bkrepo.common.artifact.stream.Range import com.tencent.bkrepo.common.metadata.constant.FAKE_SHA256 import com.tencent.bkrepo.common.metadata.constant.ID -import com.tencent.bkrepo.common.metadata.dao.RBlockNodeDao +import com.tencent.bkrepo.common.metadata.dao.blocknode.RBlockNodeDao import com.tencent.bkrepo.common.metadata.model.TBlockNode import com.tencent.bkrepo.common.metadata.service.blocknode.RBlockNodeService +import com.tencent.bkrepo.common.metadata.service.file.RFileReferenceService import com.tencent.bkrepo.common.metadata.util.BlockNodeQueryHelper import com.tencent.bkrepo.common.storage.credentials.StorageCredentials import com.tencent.bkrepo.common.storage.pojo.RegionResource @@ -45,13 +46,14 @@ import org.springframework.data.mongodb.core.query.isEqualTo import java.time.LocalDateTime abstract class RAbstractBlockNodeService( - private val rBlockNodeDao: RBlockNodeDao + private val rBlockNodeDao: RBlockNodeDao, + private val rFileReferenceService: RFileReferenceService ) : RBlockNodeService { override suspend fun createBlock(blockNode: TBlockNode, storageCredentials: StorageCredentials?): TBlockNode { with(blockNode) { val bn = rBlockNodeDao.save(blockNode) - incFileRef(bn.sha256, storageCredentials?.key) + rFileReferenceService.increment(bn.sha256, storageCredentials?.key) logger.info("Create block node[$projectId/$repoName$nodeFullPath-$startPos] ,sha256[$sha256] success.") return bn } @@ -131,8 +133,6 @@ abstract class RAbstractBlockNodeService( } } - abstract suspend fun incFileRef(sha256: String, credentialsKey: String?) - abstract suspend fun getNodeDetail(projectId: String, repoName: String, fullPath: String): NodeDetail companion object { diff --git a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/file/FileReferenceService.kt b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/file/FileReferenceService.kt similarity index 74% rename from src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/file/FileReferenceService.kt rename to src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/file/FileReferenceService.kt index 090cd8796f..1269f1a97f 100644 --- a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/file/FileReferenceService.kt +++ b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/file/FileReferenceService.kt @@ -29,31 +29,14 @@ * SOFTWARE. */ -package com.tencent.bkrepo.repository.service.file +package com.tencent.bkrepo.common.metadata.service.file -import com.tencent.bkrepo.repository.model.TNode -import com.tencent.bkrepo.repository.model.TRepository -import com.tencent.bkrepo.repository.pojo.file.FileReference +import com.tencent.bkrepo.common.metadata.pojo.file.FileReference /** * 文件引用服务接口 */ interface FileReferenceService { - /** - * 增加文件sha256的引用数量 - * - * sha256为[node]的属性，[repository]不为空则取credentialsKey属性，否则从[node]中取出repoName后从数据库查询 - * 增加引用成功则返回`true`; 如果[node]为目录，返回`false` - */ - fun increment(node: TNode, repository: TRepository? = null): Boolean - - /** - * 减少sha256的引用数量 - * - * sha256为[node]的属性，[repository]不为空则取credentialsKey属性，否则从[node]中取出repoName后从数据库查询 - * 减少引用成功则返回`true`; 如果[node]为目录，返回`false`; 如果当前sha256的引用已经为0，返回`false` - */ - fun decrement(node: TNode, repository: TRepository? = null): Boolean /** * 增加文件[sha256]在存储实例[credentialsKey]上的引用数量 diff --git a/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/file/RFileReferenceService.kt b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/file/RFileReferenceService.kt new file mode 100644 index 0000000000..d9fe3bb77c --- /dev/null +++ b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/file/RFileReferenceService.kt @@ -0,0 +1,81 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package com.tencent.bkrepo.common.metadata.service.file + +import com.tencent.bkrepo.common.metadata.pojo.file.FileReference + +/** + * 文件引用服务接口 + */ +interface RFileReferenceService { + + /** + * 增加文件[sha256]在存储实例[credentialsKey]上的引用数量 + * + * [credentialsKey]为`null`则使用默认的存储实例 + * 增加引用成功则返回`true` + */ + suspend fun increment(sha256: String, credentialsKey: String?, inc: Long = 1L): Boolean + + /** + * 减少文件[sha256]在存储实例[credentialsKey]上的文件数量 + * + * [credentialsKey]为`null`则使用默认的存储实例 + * 减少引用成功则返回`true`，如果当前[sha256]的引用已经为0，返回`false` + */ + suspend fun decrement(sha256: String, credentialsKey: String?): Boolean + + /** + * 统计文件[sha256]在存储实例[credentialsKey]上的文件引用数量 + * + * [credentialsKey]为`null`则使用默认的存储实例 + */ + suspend fun count(sha256: String, credentialsKey: String?): Long + + /** + * 获取文件文件引用信息 + * + * @param credentialsKey 文件所在存储实例 + * @param sha256 所引用文件的sha256 + */ + suspend fun get(credentialsKey: String?, sha256: String): FileReference + + /** + * 判断引用是否存在 + * + * @param sha256 所引用文件的sha256 + * @param credentialsKey 文件所在存储实例 + * + * @return 是否存在 + */ + suspend fun exists(sha256: String, credentialsKey: String?): Boolean +} diff --git a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/file/impl/FileReferenceServiceImpl.kt b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/file/impl/FileReferenceServiceImpl.kt similarity index 61% rename from src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/file/impl/FileReferenceServiceImpl.kt rename to src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/file/impl/FileReferenceServiceImpl.kt index cc971cca60..a84d42d6c9 100644 --- a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/file/impl/FileReferenceServiceImpl.kt +++ b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/file/impl/FileReferenceServiceImpl.kt @@ -29,21 +29,19 @@ * SOFTWARE. */ -package com.tencent.bkrepo.repository.service.file.impl +package com.tencent.bkrepo.common.metadata.service.file.impl import com.tencent.bkrepo.common.api.exception.NotFoundException import com.tencent.bkrepo.common.artifact.message.ArtifactMessageCode -import com.tencent.bkrepo.repository.dao.FileReferenceDao -import com.tencent.bkrepo.repository.dao.RepositoryDao -import com.tencent.bkrepo.repository.model.TFileReference -import com.tencent.bkrepo.repository.model.TNode -import com.tencent.bkrepo.repository.model.TRepository -import com.tencent.bkrepo.repository.pojo.file.FileReference -import com.tencent.bkrepo.repository.service.file.FileReferenceService +import com.tencent.bkrepo.common.metadata.condition.SyncCondition +import com.tencent.bkrepo.common.metadata.dao.file.FileReferenceDao +import com.tencent.bkrepo.common.metadata.model.TFileReference +import com.tencent.bkrepo.common.metadata.service.file.FileReferenceService +import com.tencent.bkrepo.common.metadata.util.FileReferenceQueryHelper.buildQuery +import com.tencent.bkrepo.common.metadata.pojo.file.FileReference import org.slf4j.LoggerFactory +import org.springframework.context.annotation.Conditional import org.springframework.dao.DuplicateKeyException -import org.springframework.data.mongodb.core.query.Criteria -import org.springframework.data.mongodb.core.query.Query import org.springframework.data.mongodb.core.query.Update import org.springframework.stereotype.Service @@ -51,33 +49,11 @@ import org.springframework.stereotype.Service * 文件引用服务实现类 */ @Service +@Conditional(SyncCondition::class) class FileReferenceServiceImpl( private val fileReferenceDao: FileReferenceDao, - private val repositoryDao: RepositoryDao ) : FileReferenceService { - override fun increment(node: TNode, repository: TRepository?): Boolean { - if (!validateParameter(node)) return false - return try { - val credentialsKey = findCredentialsKey(node, repository) - increment(node.sha256!!, credentialsKey) - } catch (exception: IllegalArgumentException) { - logger.error("Failed to increment reference of node [$node], repository not found.") - false - } - } - - override fun decrement(node: TNode, repository: TRepository?): Boolean { - if (!validateParameter(node)) return false - return try { - val credentialsKey = findCredentialsKey(node, repository) - decrement(node.sha256!!, credentialsKey) - } catch (exception: IllegalArgumentException) { - logger.error("Failed to decrement reference of node [$node], repository not found.") - false - } - } - override fun increment(sha256: String, credentialsKey: String?, inc: Long): Boolean { val query = buildQuery(sha256, credentialsKey) val update = Update().inc(TFileReference::count.name, inc) @@ -92,10 +68,7 @@ class FileReferenceServiceImpl( } override fun decrement(sha256: String, credentialsKey: String?): Boolean { - val criteria = Criteria.where(TFileReference::sha256.name).`is`(sha256) - criteria.and(TFileReference::credentialsKey.name).`is`(credentialsKey) - criteria.and(TFileReference::count.name).gt(0) - val query = Query(criteria) + val query = buildQuery(sha256, credentialsKey, 0) val update = Update().apply { inc(TFileReference::count.name, -1) } val result = fileReferenceDao.updateFirst(query, update) @@ -136,30 +109,6 @@ class FileReferenceServiceImpl( return tFileReference.run { FileReference(sha256, credentialsKey, count) } } - private fun findCredentialsKey(node: TNode, repository: TRepository?): String? { - if (repository != null) { - return repository.credentialsKey - } - val tRepository = repositoryDao.findByNameAndType(node.projectId, node.repoName) - require(tRepository != null) - return tRepository.credentialsKey - } - - private fun buildQuery(sha256: String, credentialsKey: String?): Query { - val criteria = Criteria.where(TFileReference::sha256.name).`is`(sha256) - criteria.and(TFileReference::credentialsKey.name).`is`(credentialsKey) - return Query.query(criteria) - } - - private fun validateParameter(node: TNode): Boolean { - if (node.folder) return false - if (node.sha256.isNullOrBlank()) { - logger.warn("Failed to change file reference, node[$node] sha256 is null or blank.") - return false - } - return true - } - companion object { private val logger = LoggerFactory.getLogger(FileReferenceServiceImpl::class.java) } diff --git a/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/file/impl/RFileReferenceServiceImpl.kt b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/file/impl/RFileReferenceServiceImpl.kt new file mode 100644 index 0000000000..c94bb46b40 --- /dev/null +++ b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/file/impl/RFileReferenceServiceImpl.kt @@ -0,0 +1,115 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package com.tencent.bkrepo.common.metadata.service.file.impl + +import com.tencent.bkrepo.common.api.exception.NotFoundException +import com.tencent.bkrepo.common.artifact.message.ArtifactMessageCode +import com.tencent.bkrepo.common.metadata.condition.ReactiveCondition +import com.tencent.bkrepo.common.metadata.dao.file.RFileReferenceDao +import com.tencent.bkrepo.common.metadata.model.TFileReference +import com.tencent.bkrepo.common.metadata.service.file.RFileReferenceService +import com.tencent.bkrepo.common.metadata.util.FileReferenceQueryHelper.buildQuery +import com.tencent.bkrepo.common.metadata.pojo.file.FileReference +import org.slf4j.LoggerFactory +import org.springframework.context.annotation.Conditional +import org.springframework.dao.DuplicateKeyException +import org.springframework.data.mongodb.core.query.Update +import org.springframework.stereotype.Service + +/** + * 文件引用服务实现类 + */ +@Service +@Conditional(ReactiveCondition::class) +class RFileReferenceServiceImpl( + private val fileReferenceDao: RFileReferenceDao, +) : RFileReferenceService { + + override suspend fun increment(sha256: String, credentialsKey: String?, inc: Long): Boolean { + val query = buildQuery(sha256, credentialsKey) + val update = Update().inc(TFileReference::count.name, inc) + try { + fileReferenceDao.upsert(query, update) + } catch (exception: DuplicateKeyException) { + // retry because upsert operation is not atomic + fileReferenceDao.upsert(query, update) + } + logger.info("Increment reference of file [$sha256] on credentialsKey [$credentialsKey].") + return true + } + + override suspend fun decrement(sha256: String, credentialsKey: String?): Boolean { + val query = buildQuery(sha256, credentialsKey, 0) + val update = Update().apply { inc(TFileReference::count.name, -1) } + val result = fileReferenceDao.updateFirst(query, update) + + if (result.modifiedCount == 1L) { + logger.info("Decrement references of file [$sha256] on credentialsKey [$credentialsKey].") + return true + } + + fileReferenceDao.findOne(buildQuery(sha256, credentialsKey)) ?: run { + logger.error("Failed to decrement reference of file [$sha256] on credentialsKey [$credentialsKey]") + return false + } + + logger.error( + "Failed to decrement reference of file [$sha256] on credentialsKey [$credentialsKey]: " + + "reference count is 0." + ) + return false + } + + override suspend fun count(sha256: String, credentialsKey: String?): Long { + val query = buildQuery(sha256, credentialsKey) + return fileReferenceDao.findOne(query)?.count ?: 0 + } + + override suspend fun get(credentialsKey: String?, sha256: String): FileReference { + val query = buildQuery(sha256, credentialsKey) + val tFileReference = fileReferenceDao.findOne(query) + ?: throw NotFoundException(ArtifactMessageCode.NODE_NOT_FOUND) + return convert(tFileReference) + } + + override suspend fun exists(sha256: String, credentialsKey: String?): Boolean { + return fileReferenceDao.exists(buildQuery(sha256, credentialsKey)) + } + + private fun convert(tFileReference: TFileReference): FileReference { + return tFileReference.run { FileReference(sha256, credentialsKey, count) } + } + + companion object { + private val logger = LoggerFactory.getLogger(RFileReferenceServiceImpl::class.java) + } +} diff --git a/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/util/FileReferenceQueryHelper.kt b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/util/FileReferenceQueryHelper.kt new file mode 100644 index 0000000000..d5f924b1ea --- /dev/null +++ b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/util/FileReferenceQueryHelper.kt @@ -0,0 +1,15 @@ +package com.tencent.bkrepo.common.metadata.util + +import com.tencent.bkrepo.common.metadata.model.TFileReference +import org.springframework.data.mongodb.core.query.Criteria +import org.springframework.data.mongodb.core.query.Query + +object FileReferenceQueryHelper { + + fun buildQuery(sha256: String, credentialsKey: String?, gt: Int? = null): Query { + val criteria = Criteria.where(TFileReference::sha256.name).`is`(sha256) + criteria.and(TFileReference::credentialsKey.name).`is`(credentialsKey) + gt?.let { criteria.and(TFileReference::count.name).gt(gt) } + return Query.query(criteria) + } +} diff --git a/src/backend/common/common-mongo-reactive/src/main/kotlin/com/tencent/bkrepo/common/mongo/reactive/dao/AbstractMongoReactiveDao.kt b/src/backend/common/common-mongo-reactive/src/main/kotlin/com/tencent/bkrepo/common/mongo/reactive/dao/AbstractMongoReactiveDao.kt index 6bac456359..8580c5d450 100644 --- a/src/backend/common/common-mongo-reactive/src/main/kotlin/com/tencent/bkrepo/common/mongo/reactive/dao/AbstractMongoReactiveDao.kt +++ b/src/backend/common/common-mongo-reactive/src/main/kotlin/com/tencent/bkrepo/common/mongo/reactive/dao/AbstractMongoReactiveDao.kt @@ -67,6 +67,15 @@ abstract class AbstractMongoReactiveDao : MongoReactiveDao { .collectList().awaitSingle() } + override suspend fun updateFirst(query: Query, update: Update): UpdateResult { + if (logger.isDebugEnabled) { + logger.debug("Mongo Dao updateFirst: [$query], [$update]") + } + return determineReactiveMongoOperations() + .updateFirst(query, update, determineCollectionName(query)) + .awaitSingle() + } + override suspend fun updateMulti(query: Query, update: Update): UpdateResult { if (logger.isDebugEnabled) { logger.debug("Mongo Dao updateMulti: [$query], [$update]") @@ -124,6 +133,13 @@ abstract class AbstractMongoReactiveDao : MongoReactiveDao { return mongoOperations.count(query, collectName).awaitSingle() } + override suspend fun exists(query: Query): Boolean { + if (logger.isDebugEnabled) { + logger.debug("Mongo Dao exists: [$query]") + } + return determineReactiveMongoOperations().exists(query, determineCollectionName(query)).awaitSingle() + } + protected open fun determineCollectionName(): String { var collectionName: String? = null if (classType.isAnnotationPresent(Document::class.java)) { diff --git a/src/backend/common/common-mongo-reactive/src/main/kotlin/com/tencent/bkrepo/common/mongo/reactive/dao/MongoReactiveDao.kt b/src/backend/common/common-mongo-reactive/src/main/kotlin/com/tencent/bkrepo/common/mongo/reactive/dao/MongoReactiveDao.kt index 4b107f432e..36ec85be50 100644 --- a/src/backend/common/common-mongo-reactive/src/main/kotlin/com/tencent/bkrepo/common/mongo/reactive/dao/MongoReactiveDao.kt +++ b/src/backend/common/common-mongo-reactive/src/main/kotlin/com/tencent/bkrepo/common/mongo/reactive/dao/MongoReactiveDao.kt @@ -52,6 +52,11 @@ interface MongoReactiveDao { */ suspend fun save(entity: E): E + /** + * 更新单条文档 + */ + suspend fun updateFirst(query: Query, update: Update): UpdateResult + /** * 更新文档 */ @@ -71,4 +76,9 @@ interface MongoReactiveDao { * count */ suspend fun count(query: Query): Long + + /** + * 判断文档是否存在 + */ + suspend fun exists(query: Query): Boolean } diff --git a/src/backend/fs/boot-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/api/RRepositoryClient.kt b/src/backend/fs/boot-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/api/RRepositoryClient.kt index 13075238f7..4d87fe640d 100644 --- a/src/backend/fs/boot-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/api/RRepositoryClient.kt +++ b/src/backend/fs/boot-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/api/RRepositoryClient.kt @@ -105,12 +105,6 @@ interface RRepositoryClient { @RequestParam type: String? = null ): Mono> - @PutMapping("/fileReference/decrement") - fun decrement(@RequestParam sha256: String, @RequestParam credentialsKey: String?): Mono> - - @PutMapping("/fileReference/increment") - fun increment(@RequestParam sha256: String, @RequestParam credentialsKey: String?): Mono> - @PostMapping("/metadata/save") fun saveMetadata(@RequestBody request: MetadataSaveRequest): Mono> diff --git a/src/backend/fs/boot-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/service/BlockNodeServiceImpl.kt b/src/backend/fs/boot-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/service/BlockNodeServiceImpl.kt index 2523c43a5e..063c47ee0e 100644 --- a/src/backend/fs/boot-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/service/BlockNodeServiceImpl.kt +++ b/src/backend/fs/boot-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/service/BlockNodeServiceImpl.kt @@ -28,8 +28,9 @@ package com.tencent.bkrepo.fs.server.service import com.tencent.bkrepo.common.artifact.exception.NodeNotFoundException -import com.tencent.bkrepo.common.metadata.dao.RBlockNodeDao +import com.tencent.bkrepo.common.metadata.dao.blocknode.RBlockNodeDao import com.tencent.bkrepo.common.metadata.service.blocknode.impl.RAbstractBlockNodeService +import com.tencent.bkrepo.common.metadata.service.file.RFileReferenceService import com.tencent.bkrepo.fs.server.api.RRepositoryClient import com.tencent.bkrepo.repository.pojo.node.NodeDetail import kotlinx.coroutines.reactor.awaitSingle @@ -39,12 +40,9 @@ import kotlinx.coroutines.reactor.awaitSingle * */ class BlockNodeServiceImpl( rBlockNodeDao: RBlockNodeDao, - private val rRepositoryClient: RRepositoryClient -) : RAbstractBlockNodeService(rBlockNodeDao) { - - override suspend fun incFileRef(sha256: String, credentialsKey: String?) { - rRepositoryClient.increment(sha256, credentialsKey).awaitSingle() - } + fileReferenceService: RFileReferenceService, + private val rRepositoryClient: RRepositoryClient, +) : RAbstractBlockNodeService(rBlockNodeDao, fileReferenceService) { override suspend fun getNodeDetail(projectId: String, repoName: String, fullPath: String): NodeDetail { return rRepositoryClient.getNodeDetail(projectId, repoName, fullPath).awaitSingle().data diff --git a/src/backend/fs/boot-fs-server/src/test/kotlin/com/tencent/com/bkrepo/fs/service/BlockNodeServiceTest.kt b/src/backend/fs/boot-fs-server/src/test/kotlin/com/tencent/com/bkrepo/fs/service/BlockNodeServiceTest.kt index 6e408e410b..33ab94ebb5 100644 --- a/src/backend/fs/boot-fs-server/src/test/kotlin/com/tencent/com/bkrepo/fs/service/BlockNodeServiceTest.kt +++ b/src/backend/fs/boot-fs-server/src/test/kotlin/com/tencent/com/bkrepo/fs/service/BlockNodeServiceTest.kt @@ -4,7 +4,7 @@ import com.tencent.bkrepo.common.api.constant.StringPool import com.tencent.bkrepo.common.api.message.CommonMessageCode import com.tencent.bkrepo.common.api.pojo.Response import com.tencent.bkrepo.common.artifact.stream.Range -import com.tencent.bkrepo.common.metadata.dao.RBlockNodeDao +import com.tencent.bkrepo.common.metadata.dao.blocknode.RBlockNodeDao import com.tencent.bkrepo.common.metadata.model.TBlockNode import com.tencent.bkrepo.common.metadata.service.blocknode.RBlockNodeService import com.tencent.bkrepo.common.storage.credentials.FileSystemCredentials @@ -21,13 +21,12 @@ import org.junit.jupiter.api.DisplayName import org.junit.jupiter.api.Test import org.mockito.Mockito import org.mockito.kotlin.any -import org.mockito.kotlin.anyOrNull import org.springframework.beans.factory.annotation.Autowired import org.springframework.boot.SpringBootConfiguration import org.springframework.boot.autoconfigure.EnableAutoConfiguration import org.springframework.boot.test.autoconfigure.data.mongo.DataMongoTest import org.springframework.boot.test.mock.mockito.MockBean -import org.springframework.context.annotation.Import +import org.springframework.context.annotation.ComponentScan import org.springframework.data.mongodb.core.query.Query import org.springframework.data.mongodb.core.query.isEqualTo import org.springframework.data.mongodb.core.query.where @@ -36,9 +35,9 @@ import reactor.core.publisher.Mono import java.time.LocalDateTime @DataMongoTest -@Import(RBlockNodeDao::class) @SpringBootConfiguration @EnableAutoConfiguration +@ComponentScan("com.tencent.bkrepo.common.metadata") @TestPropertySource(locations = ["classpath:bootstrap-ut.properties"]) class BlockNodeServiceTest { @@ -62,15 +61,9 @@ class BlockNodeServiceTest { @Test fun testCreateBlockNode() { runBlocking { - var ref = 0 - Mockito.`when`(rRepositoryClient.increment(any(), anyOrNull())).then { - ref++ - Mono.just(successResponse(true)) - } val bn = createBlockNode() Assertions.assertNotNull(bn) Assertions.assertNotNull(bn.id) - Assertions.assertEquals(1, ref) } } @@ -79,8 +72,6 @@ class BlockNodeServiceTest { fun testListRangeBlockNodes() { runBlocking { val createdDate = LocalDateTime.now().minusSeconds(1).toString() - Mockito.`when`(rRepositoryClient.increment(any(), anyOrNull())) - .thenReturn(Mono.just(successResponse(true))) createBlockNode(10) createBlockNode(20) createBlockNode(30) @@ -103,8 +94,6 @@ class BlockNodeServiceTest { fun testDeleteBlocks() { runBlocking { val createdDate = LocalDateTime.now().minusSeconds(1).toString() - Mockito.`when`(rRepositoryClient.increment(any(), anyOrNull())) - .thenReturn(Mono.just(successResponse(true))) createBlockNode(startPos = 10) createBlockNode(startPos = 20) val blocks0 = blockNodeService.listBlocks( @@ -138,8 +127,6 @@ class BlockNodeServiceTest { fun testMoveNode() { runBlocking { val createdDate = LocalDateTime.now().minusSeconds(1).toString() - Mockito.`when`(rRepositoryClient.increment(any(), anyOrNull())) - .thenReturn(Mono.just(successResponse(true))) val nodeInfo = NodeInfo( createdBy = UT_USER, createdDate = createdDate, diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/task/archive/IdleNodeArchiveJob.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/task/archive/IdleNodeArchiveJob.kt index e9769d0f3d..baf634e60a 100644 --- a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/task/archive/IdleNodeArchiveJob.kt +++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/task/archive/IdleNodeArchiveJob.kt @@ -31,6 +31,7 @@ import com.tencent.bkrepo.archive.api.ArchiveClient import com.tencent.bkrepo.archive.constant.ArchiveStorageClass import com.tencent.bkrepo.archive.request.CreateArchiveFileRequest import com.tencent.bkrepo.common.metadata.constant.FAKE_SHA256 +import com.tencent.bkrepo.common.metadata.service.file.FileReferenceService import com.tencent.bkrepo.common.mongo.dao.util.sharding.HashShardingUtils import com.tencent.bkrepo.job.SHARDING_COUNT import com.tencent.bkrepo.job.batch.base.MongoDbBatchJob @@ -38,11 +39,7 @@ import com.tencent.bkrepo.job.batch.context.NodeContext import com.tencent.bkrepo.job.batch.utils.RepositoryCommonUtils import com.tencent.bkrepo.job.batch.utils.TimeUtils import com.tencent.bkrepo.job.config.properties.IdleNodeArchiveJobProperties -import com.tencent.bkrepo.repository.api.FileReferenceClient import com.tencent.bkrepo.repository.constant.SYSTEM_USER -import java.time.Duration -import java.time.LocalDateTime -import java.util.concurrent.ConcurrentHashMap import org.slf4j.LoggerFactory import org.springframework.boot.context.properties.EnableConfigurationProperties import org.springframework.data.mongodb.core.query.Criteria @@ -50,6 +47,9 @@ import org.springframework.data.mongodb.core.query.Query import org.springframework.data.mongodb.core.query.inValues import org.springframework.data.mongodb.core.query.isEqualTo import org.springframework.stereotype.Component +import java.time.Duration +import java.time.LocalDateTime +import java.util.concurrent.ConcurrentHashMap import kotlin.reflect.KClass /** @@ -65,7 +65,7 @@ import kotlin.reflect.KClass class IdleNodeArchiveJob( private val properties: IdleNodeArchiveJobProperties, private val archiveClient: ArchiveClient, - private val fileReferenceClient: FileReferenceClient, + private val fileReferenceService: FileReferenceService, ) : MongoDbBatchJob(properties) { private var lastCutoffTime: LocalDateTime? = null private var tempCutoffTime: LocalDateTime? = null @@ -155,7 +155,7 @@ class IdleNodeArchiveJob( } val af = archiveClient.get(sha256, credentialsKey).data if (af == null) { - val count = fileReferenceClient.count(sha256, credentialsKey).data + val count = fileReferenceService.count(sha256, credentialsKey) // 归档任务不存在 if (count == 1L) { // 快速归档 diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/task/clean/DeletedBlockNodeCleanupJob.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/task/clean/DeletedBlockNodeCleanupJob.kt index 531c9a294a..b5652095c2 100644 --- a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/task/clean/DeletedBlockNodeCleanupJob.kt +++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/task/clean/DeletedBlockNodeCleanupJob.kt @@ -27,6 +27,7 @@ package com.tencent.bkrepo.job.batch.task.clean +import com.tencent.bkrepo.common.metadata.service.file.FileReferenceService import com.tencent.bkrepo.common.mongo.constant.ID import com.tencent.bkrepo.job.SHARDING_COUNT import com.tencent.bkrepo.job.batch.base.DefaultContextMongoDbJob @@ -34,7 +35,6 @@ import com.tencent.bkrepo.job.batch.base.JobContext import com.tencent.bkrepo.job.batch.utils.RepositoryCommonUtils import com.tencent.bkrepo.job.batch.utils.TimeUtils import com.tencent.bkrepo.job.config.properties.DeletedBlockNodeCleanupJobProperties -import com.tencent.bkrepo.repository.api.FileReferenceClient import org.slf4j.LoggerFactory import org.springframework.boot.context.properties.EnableConfigurationProperties import org.springframework.data.mongodb.core.query.Criteria @@ -53,7 +53,7 @@ import kotlin.reflect.KClass @EnableConfigurationProperties(DeletedBlockNodeCleanupJobProperties::class) class DeletedBlockNodeCleanupJob( private val properties: DeletedBlockNodeCleanupJobProperties, - private val fileReferenceClient: FileReferenceClient + private val fileReferenceService: FileReferenceService ) : DefaultContextMongoDbJob(properties) { data class BlockNode( @@ -106,7 +106,7 @@ class DeletedBlockNodeCleanupJob( with(blockNode) { val credentialsKey = RepositoryCommonUtils.getRepositoryDetail(projectId, repoName) .storageCredentials?.key - fileReferenceClient.decrement(sha256, credentialsKey).data + fileReferenceService.decrement(sha256, credentialsKey) } } companion object { diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/task/other/NodeCopyJob.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/task/other/NodeCopyJob.kt index 1e1b2a1564..ee6ee51c91 100644 --- a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/task/other/NodeCopyJob.kt +++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/task/other/NodeCopyJob.kt @@ -27,6 +27,7 @@ package com.tencent.bkrepo.job.batch.task.other +import com.tencent.bkrepo.common.metadata.service.file.FileReferenceService import com.tencent.bkrepo.common.mongo.constant.ID import com.tencent.bkrepo.common.service.log.LoggerHolder import com.tencent.bkrepo.common.storage.core.StorageService @@ -37,7 +38,6 @@ import com.tencent.bkrepo.job.batch.context.NodeCopyJobContext import com.tencent.bkrepo.job.batch.utils.RepositoryCommonUtils import com.tencent.bkrepo.job.config.properties.NodeCopyJobProperties import com.tencent.bkrepo.job.exception.JobExecuteException -import com.tencent.bkrepo.repository.api.FileReferenceClient import com.tencent.bkrepo.repository.constant.DEFAULT_STORAGE_CREDENTIALS_KEY import org.springframework.boot.context.properties.EnableConfigurationProperties import org.springframework.data.mongodb.core.query.Criteria @@ -52,7 +52,7 @@ import kotlin.reflect.KClass @EnableConfigurationProperties(NodeCopyJobProperties::class) class NodeCopyJob( private val storageService: StorageService, - private val fileReferenceClient: FileReferenceClient, + private val fileReferenceService: FileReferenceService, properties: NodeCopyJobProperties ) : MongoDbBatchJob(properties) { @@ -142,12 +142,12 @@ class NodeCopyJob( ) { var dstCredentialsKey: String? = dstCredentials?.key ?: DEFAULT_STORAGE_CREDENTIALS_KEY if (dstCredentialsKey != node.copyIntoCredentialsKey) { - fileReferenceClient.decrement(digest, node.copyIntoCredentialsKey) + fileReferenceService.decrement(digest, node.copyIntoCredentialsKey) if (dstCredentialsKey == DEFAULT_STORAGE_CREDENTIALS_KEY) { // 还原为默认存储key为null dstCredentialsKey = null } - fileReferenceClient.increment(digest, dstCredentialsKey) + fileReferenceService.increment(digest, dstCredentialsKey) } } diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/task/storage/StorageReconcileJob.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/task/storage/StorageReconcileJob.kt index d1e414fa6a..b8827412af 100644 --- a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/task/storage/StorageReconcileJob.kt +++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/task/storage/StorageReconcileJob.kt @@ -3,6 +3,7 @@ package com.tencent.bkrepo.job.batch.task.storage import com.google.common.hash.BloomFilter import com.google.common.hash.Funnels import com.tencent.bkrepo.common.api.constant.StringPool +import com.tencent.bkrepo.common.metadata.service.file.FileReferenceService import com.tencent.bkrepo.common.service.cluster.properties.ClusterProperties import com.tencent.bkrepo.common.storage.core.FileStorage import com.tencent.bkrepo.common.storage.core.StorageProperties @@ -14,7 +15,6 @@ import com.tencent.bkrepo.job.batch.base.JobContext import com.tencent.bkrepo.job.batch.utils.NodeCommonUtils import com.tencent.bkrepo.job.config.properties.FileReferenceCleanupJobProperties import com.tencent.bkrepo.job.config.properties.StorageReconcileJobProperties -import com.tencent.bkrepo.repository.api.FileReferenceClient import com.tencent.bkrepo.repository.api.StorageCredentialsClient import org.slf4j.LoggerFactory import org.springframework.boot.context.properties.EnableConfigurationProperties @@ -40,7 +40,7 @@ class StorageReconcileJob( private val clusterProperties: ClusterProperties, private val storageCredentialsClient: StorageCredentialsClient, private val storageProperties: StorageProperties, - private val fileReferenceClient: FileReferenceClient, + private val fileReferenceService: FileReferenceService, ) : DefaultContextJob(properties) { override fun doStart0(jobContext: JobContext) { // 校验默认存储 @@ -76,10 +76,10 @@ class StorageReconcileJob( } // 二次确认,因为待确认的文件数量远低于总体数量，所以这里采用直接查表，效率更高些。 pendingDeleteList.forEach { - val exists = fileReferenceClient.exists(it, credentialsKey).data ?: true + val exists = fileReferenceService.exists(it, credentialsKey) if (!exists) { logger.info("Delete file [$it]") - fileReferenceClient.increment(it, credentialsKey, 0) + fileReferenceService.increment(it, credentialsKey, 0) deleted++ } } diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/migrate/executor/BaseTaskExecutor.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/migrate/executor/BaseTaskExecutor.kt index 9fa97b33f2..084b4d232e 100644 --- a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/migrate/executor/BaseTaskExecutor.kt +++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/migrate/executor/BaseTaskExecutor.kt @@ -29,6 +29,7 @@ package com.tencent.bkrepo.job.migrate.executor import com.tencent.bkrepo.common.api.util.HumanReadable import com.tencent.bkrepo.common.metadata.constant.FAKE_SHA256 +import com.tencent.bkrepo.common.metadata.service.file.FileReferenceService import com.tencent.bkrepo.common.service.actuator.ActuatorConfiguration.Companion.SERVICE_INSTANCE_ID import com.tencent.bkrepo.common.storage.core.StorageService import com.tencent.bkrepo.common.storage.monitor.measureThroughput @@ -46,7 +47,6 @@ import com.tencent.bkrepo.job.migrate.pojo.MigrateRepoStorageTaskState.PENDING import com.tencent.bkrepo.job.migrate.pojo.MigrationContext import com.tencent.bkrepo.job.migrate.pojo.Node import com.tencent.bkrepo.job.migrate.utils.ExecutingTaskRecorder -import com.tencent.bkrepo.repository.api.FileReferenceClient import org.slf4j.LoggerFactory import org.springframework.beans.factory.annotation.Value import java.util.concurrent.RejectedExecutionException @@ -57,7 +57,7 @@ abstract class BaseTaskExecutor( protected val properties: MigrateRepoStorageProperties, protected val migrateRepoStorageTaskDao: MigrateRepoStorageTaskDao, protected val migrateFailedNodeDao: MigrateFailedNodeDao, - private val fileReferenceClient: FileReferenceClient, + private val fileReferenceService: FileReferenceService, private val storageService: StorageService, private val executingTaskRecorder: ExecutingTaskRecorder, ) : TaskExecutor { @@ -167,7 +167,7 @@ abstract class BaseTaskExecutor( return } - if (fileReferenceClient.count(sha256, context.task.dstStorageKey).data!! > 0) { + if (fileReferenceService.count(sha256, context.task.dstStorageKey) > 0) { /* 可能由于在上传制品时使用的旧存储，而创建Node时由于会重新查一遍仓库的存储凭据而使用新存储 这种情况会导致目标存储引用大于0但是文件不再目标存储，此时仅迁移存储不修改引用数 @@ -192,11 +192,11 @@ abstract class BaseTaskExecutor( // FileReferenceCleanupJob 会定期清理引用为0的文件数据，所以不需要删除文件数据 // old引用计数 -1 - if (fileReferenceClient.decrement(sha256, srcStorageKey).data != true) { + if (!fileReferenceService.decrement(sha256, srcStorageKey)) { logger.error("Failed to decrement file reference[$sha256] on storage[$srcStorageKey].") } // new引用计数 +1 - if (fileReferenceClient.increment(sha256, dstStorageKey).data != true) { + if (!fileReferenceService.increment(sha256, dstStorageKey)) { logger.error("Failed to increment file reference[$sha256] on storage[$dstStorageKey].") } } diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/migrate/executor/CorrectExecutor.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/migrate/executor/CorrectExecutor.kt index ba6a81a098..f670e086e6 100644 --- a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/migrate/executor/CorrectExecutor.kt +++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/migrate/executor/CorrectExecutor.kt @@ -27,6 +27,7 @@ package com.tencent.bkrepo.job.migrate.executor +import com.tencent.bkrepo.common.metadata.service.file.FileReferenceService import com.tencent.bkrepo.common.storage.core.StorageService import com.tencent.bkrepo.job.migrate.config.MigrateRepoStorageProperties import com.tencent.bkrepo.job.migrate.dao.MigrateFailedNodeDao @@ -37,7 +38,6 @@ import com.tencent.bkrepo.job.migrate.utils.ExecutingTaskRecorder import com.tencent.bkrepo.job.migrate.utils.MigrateRepoStorageUtils.buildThreadPoolExecutor import com.tencent.bkrepo.job.migrate.utils.NodeIterator import com.tencent.bkrepo.job.migrate.utils.TransferDataExecutor -import com.tencent.bkrepo.repository.api.FileReferenceClient import org.slf4j.LoggerFactory import org.springframework.data.mongodb.core.MongoTemplate import org.springframework.stereotype.Component @@ -47,7 +47,7 @@ import java.util.concurrent.TimeUnit @Component class CorrectExecutor( properties: MigrateRepoStorageProperties, - fileReferenceClient: FileReferenceClient, + fileReferenceService: FileReferenceService, migrateRepoStorageTaskDao: MigrateRepoStorageTaskDao, migrateFailedNodeDao: MigrateFailedNodeDao, storageService: StorageService, @@ -59,7 +59,7 @@ class CorrectExecutor( properties, migrateRepoStorageTaskDao, migrateFailedNodeDao, - fileReferenceClient, + fileReferenceService, storageService, executingTaskRecorder, ) { diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/migrate/executor/FinishExecutor.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/migrate/executor/FinishExecutor.kt index b4bbbe26ee..951f4c54d0 100644 --- a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/migrate/executor/FinishExecutor.kt +++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/migrate/executor/FinishExecutor.kt @@ -27,6 +27,7 @@ package com.tencent.bkrepo.job.migrate.executor +import com.tencent.bkrepo.common.metadata.service.file.FileReferenceService import com.tencent.bkrepo.common.storage.core.StorageService import com.tencent.bkrepo.job.migrate.config.MigrateRepoStorageProperties import com.tencent.bkrepo.job.migrate.dao.MigrateFailedNodeDao @@ -35,7 +36,6 @@ import com.tencent.bkrepo.job.migrate.pojo.MigrateRepoStorageTaskState.FINISHING import com.tencent.bkrepo.job.migrate.pojo.MigrateRepoStorageTaskState.MIGRATE_FAILED_NODE_FINISHED import com.tencent.bkrepo.job.migrate.pojo.MigrationContext import com.tencent.bkrepo.job.migrate.utils.ExecutingTaskRecorder -import com.tencent.bkrepo.repository.api.FileReferenceClient import com.tencent.bkrepo.repository.api.RepositoryClient import org.slf4j.LoggerFactory import org.springframework.stereotype.Component @@ -43,7 +43,7 @@ import org.springframework.stereotype.Component @Component class FinishExecutor( properties: MigrateRepoStorageProperties, - fileReferenceClient: FileReferenceClient, + fileReferenceService: FileReferenceService, migrateRepoStorageTaskDao: MigrateRepoStorageTaskDao, migrateFailedNodeDao: MigrateFailedNodeDao, storageService: StorageService, @@ -53,7 +53,7 @@ class FinishExecutor( properties, migrateRepoStorageTaskDao, migrateFailedNodeDao, - fileReferenceClient, + fileReferenceService, storageService, executingTaskRecorder, ) { diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/migrate/executor/MigrateExecutor.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/migrate/executor/MigrateExecutor.kt index d974e6ecfe..3a161986b4 100644 --- a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/migrate/executor/MigrateExecutor.kt +++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/migrate/executor/MigrateExecutor.kt @@ -27,6 +27,7 @@ package com.tencent.bkrepo.job.migrate.executor +import com.tencent.bkrepo.common.metadata.service.file.FileReferenceService import com.tencent.bkrepo.common.storage.core.StorageService import com.tencent.bkrepo.job.migrate.config.MigrateRepoStorageProperties import com.tencent.bkrepo.job.migrate.dao.MigrateFailedNodeDao @@ -39,7 +40,6 @@ import com.tencent.bkrepo.job.migrate.utils.MigrateRepoStorageUtils.buildThreadP import com.tencent.bkrepo.job.migrate.utils.MigratedTaskNumberPriorityQueue import com.tencent.bkrepo.job.migrate.utils.NodeIterator import com.tencent.bkrepo.job.migrate.utils.TransferDataExecutor -import com.tencent.bkrepo.repository.api.FileReferenceClient import com.tencent.bkrepo.repository.api.RepositoryClient import org.slf4j.LoggerFactory import org.springframework.data.mongodb.core.MongoTemplate @@ -51,7 +51,7 @@ import java.util.concurrent.TimeUnit @Component class MigrateExecutor( properties: MigrateRepoStorageProperties, - fileReferenceClient: FileReferenceClient, + fileReferenceService: FileReferenceService, migrateRepoStorageTaskDao: MigrateRepoStorageTaskDao, migrateFailedNodeDao: MigrateFailedNodeDao, storageService: StorageService, @@ -64,7 +64,7 @@ class MigrateExecutor( properties, migrateRepoStorageTaskDao, migrateFailedNodeDao, - fileReferenceClient, + fileReferenceService, storageService, executingTaskRecorder, ) { diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/migrate/executor/MigrateFailedNodeExecutor.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/migrate/executor/MigrateFailedNodeExecutor.kt index 82bdcf75de..831679c2a1 100644 --- a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/migrate/executor/MigrateFailedNodeExecutor.kt +++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/migrate/executor/MigrateFailedNodeExecutor.kt @@ -27,6 +27,7 @@ package com.tencent.bkrepo.job.migrate.executor +import com.tencent.bkrepo.common.metadata.service.file.FileReferenceService import com.tencent.bkrepo.common.storage.core.StorageService import com.tencent.bkrepo.job.migrate.Constant.MAX_MIGRATE_FAILED_RETRY_TIMES import com.tencent.bkrepo.job.migrate.config.MigrateRepoStorageProperties @@ -40,7 +41,6 @@ import com.tencent.bkrepo.job.migrate.strategy.MigrateFailedNodeFixer import com.tencent.bkrepo.job.migrate.utils.ExecutingTaskRecorder import com.tencent.bkrepo.job.migrate.utils.MigrateRepoStorageUtils.buildThreadPoolExecutor import com.tencent.bkrepo.job.migrate.utils.TransferDataExecutor -import com.tencent.bkrepo.repository.api.FileReferenceClient import org.slf4j.LoggerFactory import org.springframework.stereotype.Component import java.util.concurrent.ThreadPoolExecutor @@ -49,7 +49,7 @@ import java.util.concurrent.TimeUnit @Component class MigrateFailedNodeExecutor( properties: MigrateRepoStorageProperties, - fileReferenceClient: FileReferenceClient, + fileReferenceService: FileReferenceService, migrateRepoStorageTaskDao: MigrateRepoStorageTaskDao, migrateFailedNodeDao: MigrateFailedNodeDao, storageService: StorageService, @@ -60,7 +60,7 @@ class MigrateFailedNodeExecutor( properties, migrateRepoStorageTaskDao, migrateFailedNodeDao, - fileReferenceClient, + fileReferenceService, storageService, executingTaskRecorder, ) { diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/migrate/strategy/FileNotFoundAutoFixStrategy.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/migrate/strategy/FileNotFoundAutoFixStrategy.kt index 2c05308407..67413949c9 100644 --- a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/migrate/strategy/FileNotFoundAutoFixStrategy.kt +++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/migrate/strategy/FileNotFoundAutoFixStrategy.kt @@ -28,6 +28,7 @@ package com.tencent.bkrepo.job.migrate.strategy import com.tencent.bkrepo.common.artifact.stream.Range +import com.tencent.bkrepo.common.metadata.service.file.FileReferenceService import com.tencent.bkrepo.common.mongo.dao.util.sharding.HashShardingUtils.shardingSequenceFor import com.tencent.bkrepo.common.storage.core.StorageProperties import com.tencent.bkrepo.common.storage.core.StorageService @@ -41,7 +42,6 @@ import com.tencent.bkrepo.job.migrate.dao.MigrateFailedNodeDao import com.tencent.bkrepo.job.migrate.model.TArchiveMigrateFailedNode import com.tencent.bkrepo.job.migrate.model.TMigrateFailedNode import com.tencent.bkrepo.job.migrate.pojo.Node -import com.tencent.bkrepo.repository.api.FileReferenceClient import com.tencent.bkrepo.repository.api.StorageCredentialsClient import org.slf4j.LoggerFactory import org.springframework.data.mongodb.core.MongoTemplate @@ -54,7 +54,7 @@ import org.springframework.stereotype.Component class FileNotFoundAutoFixStrategy( private val mongoTemplate: MongoTemplate, private val storageCredentialsClient: StorageCredentialsClient, - private val fileReferenceClient: FileReferenceClient, + private val fileReferenceService: FileReferenceService, private val storageService: StorageService, private val storageProperties: StorageProperties, private val migrateFailedNodeDao: MigrateFailedNodeDao, @@ -136,7 +136,7 @@ class FileNotFoundAutoFixStrategy( ais.close() // 尝试从其他存储复制到当前存储 storageService.copy(migrateFailedNode.sha256, credentials, oldCredentials) - fileReferenceClient.increment(migrateFailedNode.sha256, oldCredentials.key, 0L) + fileReferenceService.increment(migrateFailedNode.sha256, oldCredentials.key, 0L) logger.info("copy [$fullPath] from credentials[$key] success, task[$projectId/$repoName]") return true } diff --git a/src/backend/job/biz-job/src/test/kotlin/com/tencent/bkrepo/job/batch/JobBaseTest.kt b/src/backend/job/biz-job/src/test/kotlin/com/tencent/bkrepo/job/batch/JobBaseTest.kt index 374dbed539..f9492013d4 100644 --- a/src/backend/job/biz-job/src/test/kotlin/com/tencent/bkrepo/job/batch/JobBaseTest.kt +++ b/src/backend/job/biz-job/src/test/kotlin/com/tencent/bkrepo/job/batch/JobBaseTest.kt @@ -66,7 +66,7 @@ import org.springframework.test.context.TestPropertySource ], ) @ComponentScan( - basePackages = ["com.tencent.bkrepo.job"], + basePackages = ["com.tencent.bkrepo.job", "com.tencent.bkrepo.common.metadata"], excludeFilters = [ ComponentScan.Filter( type = FilterType.ASSIGNABLE_TYPE, diff --git a/src/backend/job/biz-job/src/test/kotlin/com/tencent/bkrepo/job/batch/task/storage/StorageReconcileJobTest.kt b/src/backend/job/biz-job/src/test/kotlin/com/tencent/bkrepo/job/batch/task/storage/StorageReconcileJobTest.kt index da46b102a9..d53ab3a9a2 100644 --- a/src/backend/job/biz-job/src/test/kotlin/com/tencent/bkrepo/job/batch/task/storage/StorageReconcileJobTest.kt +++ b/src/backend/job/biz-job/src/test/kotlin/com/tencent/bkrepo/job/batch/task/storage/StorageReconcileJobTest.kt @@ -2,6 +2,7 @@ package com.tencent.bkrepo.job.batch.task.storage import com.tencent.bkrepo.common.artifact.api.ArtifactFile import com.tencent.bkrepo.common.artifact.api.FileSystemArtifactFile +import com.tencent.bkrepo.common.metadata.service.file.FileReferenceService import com.tencent.bkrepo.common.service.util.ResponseBuilder import com.tencent.bkrepo.common.storage.core.StorageProperties import com.tencent.bkrepo.common.storage.core.StorageService @@ -10,7 +11,6 @@ import com.tencent.bkrepo.common.storage.util.toPath import com.tencent.bkrepo.job.batch.JobBaseTest import com.tencent.bkrepo.job.batch.utils.NodeCommonUtils import com.tencent.bkrepo.job.migrate.MigrateRepoStorageService -import com.tencent.bkrepo.repository.api.FileReferenceClient import com.tencent.bkrepo.repository.api.RepositoryClient import com.tencent.bkrepo.repository.api.StorageCredentialsClient import org.junit.jupiter.api.AfterEach @@ -42,7 +42,7 @@ class StorageReconcileJobTest @Autowired constructor( lateinit var storageCredentialsClient: StorageCredentialsClient @MockBean - lateinit var fileReferenceClient: FileReferenceClient + lateinit var fileReferenceService: FileReferenceService @MockBean lateinit var repositoryClient: RepositoryClient @@ -68,12 +68,12 @@ class StorageReconcileJobTest @Autowired constructor( @Test fun reconcileTest() { var checked = 0 - `when`(fileReferenceClient.increment(anyString(), isNull(), anyLong())).then { + `when`(fileReferenceService.increment(anyString(), isNull(), anyLong())).then { checked++ - ResponseBuilder.success(true) + true } - `when`(fileReferenceClient.exists(anyString(), isNull())).then { - ResponseBuilder.success(false) + `when`(fileReferenceService.exists(anyString(), isNull())).then { + false } repeat(10) { val file = createTempArtifactFile() diff --git a/src/backend/job/biz-job/src/test/kotlin/com/tencent/bkrepo/job/migrate/executor/CorrectExecutorTest.kt b/src/backend/job/biz-job/src/test/kotlin/com/tencent/bkrepo/job/migrate/executor/CorrectExecutorTest.kt index 25ef84526a..dedea20bfc 100644 --- a/src/backend/job/biz-job/src/test/kotlin/com/tencent/bkrepo/job/migrate/executor/CorrectExecutorTest.kt +++ b/src/backend/job/biz-job/src/test/kotlin/com/tencent/bkrepo/job/migrate/executor/CorrectExecutorTest.kt @@ -27,7 +27,6 @@ package com.tencent.bkrepo.job.migrate.executor -import com.tencent.bkrepo.common.api.pojo.Response import com.tencent.bkrepo.job.UT_PROJECT_ID import com.tencent.bkrepo.job.UT_REPO_NAME import com.tencent.bkrepo.job.migrate.pojo.MigrateRepoStorageTaskState.CORRECT_FINISHED @@ -95,7 +94,7 @@ class CorrectExecutorTest @Autowired constructor( whenever(storageService.copy(anyString(), anyOrNull(), anyOrNull())).then { throw FileNotFoundException() } - whenever(fileReferenceClient.count(anyString(), anyOrNull())).thenReturn(Response(0, data = 1L)) + whenever(fileReferenceService.count(anyString(), anyOrNull())).thenReturn(1L) // 创建node用于模拟遍历迁移 val now = LocalDateTime.now() mongoTemplate.createNode(createDate = now.plusMinutes(1L)) diff --git a/src/backend/job/biz-job/src/test/kotlin/com/tencent/bkrepo/job/migrate/executor/ExecutorBaseTest.kt b/src/backend/job/biz-job/src/test/kotlin/com/tencent/bkrepo/job/migrate/executor/ExecutorBaseTest.kt index 5668495764..5982ba4cbe 100644 --- a/src/backend/job/biz-job/src/test/kotlin/com/tencent/bkrepo/job/migrate/executor/ExecutorBaseTest.kt +++ b/src/backend/job/biz-job/src/test/kotlin/com/tencent/bkrepo/job/migrate/executor/ExecutorBaseTest.kt @@ -28,6 +28,7 @@ package com.tencent.bkrepo.job.migrate.executor import com.tencent.bkrepo.common.api.pojo.Response +import com.tencent.bkrepo.common.metadata.service.file.FileReferenceService import com.tencent.bkrepo.common.mongo.constant.ID import com.tencent.bkrepo.common.storage.core.StorageProperties import com.tencent.bkrepo.common.storage.core.StorageService @@ -46,7 +47,6 @@ import com.tencent.bkrepo.job.migrate.pojo.MigrateRepoStorageTask import com.tencent.bkrepo.job.migrate.pojo.MigrationContext import com.tencent.bkrepo.job.migrate.utils.ExecutingTaskRecorder import com.tencent.bkrepo.job.migrate.utils.MigrateTestUtils -import com.tencent.bkrepo.repository.api.FileReferenceClient import com.tencent.bkrepo.repository.api.RepositoryClient import com.tencent.bkrepo.repository.api.StorageCredentialsClient import org.junit.jupiter.api.TestInstance @@ -73,7 +73,6 @@ import java.time.LocalDateTime @Import( TaskExecutionAutoConfiguration::class, MigrateRepoStorageProperties::class, - FileReferenceClient::class, RepositoryClient::class, RepositoryCommonUtils::class, StorageProperties::class, @@ -100,7 +99,7 @@ open class ExecutorBaseTest { protected lateinit var executingTaskRecorder: ExecutingTaskRecorder @MockBean - protected lateinit var fileReferenceClient: FileReferenceClient + protected lateinit var fileReferenceService: FileReferenceService @MockBean protected lateinit var repositoryClient: RepositoryClient @@ -112,9 +111,9 @@ open class ExecutorBaseTest { protected lateinit var storageService: StorageService fun initMock() { - whenever(fileReferenceClient.increment(any(), anyOrNull(), any())).thenReturn(Response(0, data = true)) - whenever(fileReferenceClient.decrement(any(), anyOrNull())).thenReturn(Response(0, data = true)) - whenever(fileReferenceClient.count(anyString(), anyOrNull())).thenReturn(Response(0, data = 0L)) + whenever(fileReferenceService.increment(any(), anyOrNull(), any())).thenReturn(true) + whenever(fileReferenceService.decrement(any(), anyOrNull())).thenReturn(true) + whenever(fileReferenceService.count(anyString(), anyOrNull())).thenReturn(0) whenever(repositoryClient.getRepoDetail(anyString(), anyString(), anyOrNull())) .thenReturn(Response(0, "", MigrateTestUtils.buildRepo())) diff --git a/src/backend/job/biz-job/src/test/kotlin/com/tencent/bkrepo/job/migrate/strategy/FileNotFoundAutoFixStrategyTest.kt b/src/backend/job/biz-job/src/test/kotlin/com/tencent/bkrepo/job/migrate/strategy/FileNotFoundAutoFixStrategyTest.kt index dcf3601bbd..d9cae89972 100644 --- a/src/backend/job/biz-job/src/test/kotlin/com/tencent/bkrepo/job/migrate/strategy/FileNotFoundAutoFixStrategyTest.kt +++ b/src/backend/job/biz-job/src/test/kotlin/com/tencent/bkrepo/job/migrate/strategy/FileNotFoundAutoFixStrategyTest.kt @@ -4,6 +4,7 @@ import com.sun.xml.internal.messaging.saaj.util.ByteInputStream import com.tencent.bkrepo.common.api.pojo.Response import com.tencent.bkrepo.common.artifact.stream.Range import com.tencent.bkrepo.common.artifact.stream.artifactStream +import com.tencent.bkrepo.common.metadata.service.file.FileReferenceService import com.tencent.bkrepo.common.storage.core.StorageProperties import com.tencent.bkrepo.common.storage.core.StorageService import com.tencent.bkrepo.common.storage.credentials.FileSystemCredentials @@ -15,7 +16,6 @@ import com.tencent.bkrepo.job.migrate.utils.MigrateTestUtils import com.tencent.bkrepo.job.migrate.utils.MigrateTestUtils.createNode import com.tencent.bkrepo.job.migrate.utils.MigrateTestUtils.insertFailedNode import com.tencent.bkrepo.job.migrate.utils.MigrateTestUtils.removeNodes -import com.tencent.bkrepo.repository.api.FileReferenceClient import com.tencent.bkrepo.repository.api.RepositoryClient import com.tencent.bkrepo.repository.api.StorageCredentialsClient import org.junit.jupiter.api.Assertions.assertEquals @@ -35,6 +35,7 @@ import org.mockito.kotlin.whenever import org.springframework.beans.factory.annotation.Autowired import org.springframework.boot.test.autoconfigure.data.mongo.DataMongoTest import org.springframework.boot.test.mock.mockito.MockBean +import org.springframework.context.annotation.ComponentScan import org.springframework.context.annotation.Import import org.springframework.data.mongodb.core.MongoTemplate import org.springframework.data.mongodb.core.query.Query @@ -50,18 +51,20 @@ import org.springframework.test.context.TestPropertySource MigrateFailedNodeDao::class, ArchiveMigrateFailedNodeDao::class, ) +@ComponentScan(basePackages = ["com.tencent.bkrepo.common.metadata"]) @TestPropertySource(locations = ["classpath:bootstrap-ut.properties"]) class FileNotFoundAutoFixStrategyTest @Autowired constructor( private val mongoTemplate: MongoTemplate, private val strategy: FileNotFoundAutoFixStrategy, private val migrateFailedNodeDao: MigrateFailedNodeDao, private val archiveMigrateFailedNodeDao: ArchiveMigrateFailedNodeDao, + repositoryCommonUtils: RepositoryCommonUtils ) { @MockBean private lateinit var storageCredentialsClient: StorageCredentialsClient @MockBean - private lateinit var fileReferenceClient: FileReferenceClient + private lateinit var fileReferenceService: FileReferenceService @MockBean private lateinit var repositoryClient: RepositoryClient @@ -75,8 +78,8 @@ class FileNotFoundAutoFixStrategyTest @Autowired constructor( .thenReturn(Response(0, data = listOf(FileSystemCredentials()))) whenever(storageCredentialsClient.findByKey(anyString())) .thenReturn(Response(0, data = FileSystemCredentials())) - whenever(fileReferenceClient.increment(anyString(), anyOrNull(), any())) - .thenReturn(Response(0, data = true)) + whenever(fileReferenceService.increment(anyString(), anyOrNull(), any())) + .thenReturn(true) whenever(repositoryClient.getRepoDetail(anyString(), anyString(), anyOrNull())) .thenReturn(Response(0, "", MigrateTestUtils.buildRepo())) whenever(storageService.exist(anyString(), anyOrNull())).thenReturn(false) @@ -124,7 +127,7 @@ class FileNotFoundAutoFixStrategyTest @Autowired constructor( val node = mongoTemplate.createNode() val failedNode = migrateFailedNodeDao.insertFailedNode(node.fullPath) assertTrue(strategy.fix(failedNode)) - verify(fileReferenceClient, times(1)).increment(any(), anyOrNull(), any()) + verify(fileReferenceService, times(1)).increment(any(), anyOrNull(), any()) } @Test diff --git a/src/backend/repository/api-repository/src/main/kotlin/com/tencent/bkrepo/repository/api/FileReferenceClient.kt b/src/backend/repository/api-repository/src/main/kotlin/com/tencent/bkrepo/repository/api/FileReferenceClient.kt index 71542374c6..e8af9ce2ca 100644 --- a/src/backend/repository/api-repository/src/main/kotlin/com/tencent/bkrepo/repository/api/FileReferenceClient.kt +++ b/src/backend/repository/api-repository/src/main/kotlin/com/tencent/bkrepo/repository/api/FileReferenceClient.kt @@ -10,6 +10,7 @@ import org.springframework.web.bind.annotation.RequestParam @FeignClient(REPOSITORY_SERVICE_NAME, contextId = "FileReferenceClient") @RequestMapping("/service/fileReference") +@Deprecated(message = "replace for FileReferenceService or RFileReferenceService") interface FileReferenceClient { @PutMapping("/decrement") diff --git a/src/backend/repository/api-repository/src/main/kotlin/com/tencent/bkrepo/repository/api/RepositoryClient.kt b/src/backend/repository/api-repository/src/main/kotlin/com/tencent/bkrepo/repository/api/RepositoryClient.kt index 7e0d711bdb..8d1579d023 100644 --- a/src/backend/repository/api-repository/src/main/kotlin/com/tencent/bkrepo/repository/api/RepositoryClient.kt +++ b/src/backend/repository/api-repository/src/main/kotlin/com/tencent/bkrepo/repository/api/RepositoryClient.kt @@ -46,7 +46,6 @@ import io.swagger.annotations.Api import io.swagger.annotations.ApiOperation import io.swagger.annotations.ApiParam import org.springframework.cloud.openfeign.FeignClient -import org.springframework.context.annotation.Primary import org.springframework.web.bind.annotation.DeleteMapping import org.springframework.web.bind.annotation.GetMapping import org.springframework.web.bind.annotation.PathVariable @@ -59,7 +58,6 @@ import org.springframework.web.bind.annotation.RequestParam * 仓库服务接口 */ @Api("仓库服务接口") -@Primary @FeignClient(REPOSITORY_SERVICE_NAME, contextId = "RepositoryClient", primary = false) @RequestMapping("/service/repo") interface RepositoryClient { diff --git a/src/backend/repository/api-repository/src/main/kotlin/com/tencent/bkrepo/repository/api/StorageCredentialsClient.kt b/src/backend/repository/api-repository/src/main/kotlin/com/tencent/bkrepo/repository/api/StorageCredentialsClient.kt index 00d59da6c7..78a4eb0675 100644 --- a/src/backend/repository/api-repository/src/main/kotlin/com/tencent/bkrepo/repository/api/StorageCredentialsClient.kt +++ b/src/backend/repository/api-repository/src/main/kotlin/com/tencent/bkrepo/repository/api/StorageCredentialsClient.kt @@ -42,7 +42,7 @@ import org.springframework.web.bind.annotation.RequestParam /** * 存储凭证服务接口 */ -@FeignClient(REPOSITORY_SERVICE_NAME, contextId = "StorageCredentialsClient") +@FeignClient(REPOSITORY_SERVICE_NAME, contextId = "StorageCredentialsClient", primary = false) @RequestMapping("/service/storage/credentials") interface StorageCredentialsClient { diff --git a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/controller/proxy/ProxyFileReferenceController.kt b/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/controller/proxy/ProxyFileReferenceController.kt index 93086a9ecc..60a4bd42ce 100644 --- a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/controller/proxy/ProxyFileReferenceController.kt +++ b/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/controller/proxy/ProxyFileReferenceController.kt @@ -30,7 +30,7 @@ package com.tencent.bkrepo.repository.controller.proxy import com.tencent.bkrepo.common.api.pojo.Response import com.tencent.bkrepo.common.service.util.ResponseBuilder import com.tencent.bkrepo.repository.api.proxy.ProxyFileReferenceClient -import com.tencent.bkrepo.repository.service.file.FileReferenceService +import com.tencent.bkrepo.common.metadata.service.file.FileReferenceService import org.springframework.web.bind.annotation.RestController @RestController diff --git a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/controller/service/FileReferenceController.kt b/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/controller/service/FileReferenceController.kt index 20bc4f14c1..86b07f415c 100644 --- a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/controller/service/FileReferenceController.kt +++ b/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/controller/service/FileReferenceController.kt @@ -3,7 +3,7 @@ package com.tencent.bkrepo.repository.controller.service import com.tencent.bkrepo.common.api.pojo.Response import com.tencent.bkrepo.common.service.util.ResponseBuilder import com.tencent.bkrepo.repository.api.FileReferenceClient -import com.tencent.bkrepo.repository.service.file.FileReferenceService +import com.tencent.bkrepo.common.metadata.service.file.FileReferenceService import org.springframework.web.bind.annotation.RestController @RestController diff --git a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/controller/service/StorageCredentialsController.kt b/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/controller/service/StorageCredentialsController.kt index b834414bb5..49bf3e2512 100644 --- a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/controller/service/StorageCredentialsController.kt +++ b/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/controller/service/StorageCredentialsController.kt @@ -37,12 +37,14 @@ import com.tencent.bkrepo.common.storage.core.StorageProperties import com.tencent.bkrepo.common.storage.credentials.StorageCredentials import com.tencent.bkrepo.repository.api.StorageCredentialsClient import com.tencent.bkrepo.repository.service.repo.StorageCredentialService +import org.springframework.context.annotation.Primary import org.springframework.web.bind.annotation.RestController /** * 存储凭证服务接口实现类 */ @RestController +@Primary class StorageCredentialsController( private val storageCredentialService: StorageCredentialService, private val storageProperties: StorageProperties diff --git a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/controller/user/UserFileReferenceController.kt b/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/controller/user/UserFileReferenceController.kt index a23e041c45..87f0b0c36c 100644 --- a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/controller/user/UserFileReferenceController.kt +++ b/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/controller/user/UserFileReferenceController.kt @@ -33,8 +33,8 @@ import com.tencent.bkrepo.common.artifact.message.ArtifactMessageCode import com.tencent.bkrepo.common.security.permission.Principal import com.tencent.bkrepo.common.security.permission.PrincipalType import com.tencent.bkrepo.common.service.util.ResponseBuilder -import com.tencent.bkrepo.repository.pojo.file.FileReference -import com.tencent.bkrepo.repository.service.file.FileReferenceService +import com.tencent.bkrepo.common.metadata.pojo.file.FileReference +import com.tencent.bkrepo.common.metadata.service.file.FileReferenceService import com.tencent.bkrepo.repository.service.repo.RepositoryService import io.swagger.annotations.Api import io.swagger.annotations.ApiOperation diff --git a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/job/DeletedNodeCleanupJob.kt b/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/job/DeletedNodeCleanupJob.kt index 57c9433525..aa1b4f72ee 100644 --- a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/job/DeletedNodeCleanupJob.kt +++ b/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/job/DeletedNodeCleanupJob.kt @@ -38,7 +38,7 @@ import com.tencent.bkrepo.repository.dao.RepositoryDao import com.tencent.bkrepo.repository.job.base.CenterNodeJob import com.tencent.bkrepo.repository.model.TNode import com.tencent.bkrepo.repository.model.TRepository -import com.tencent.bkrepo.repository.service.file.FileReferenceService +import com.tencent.bkrepo.common.metadata.service.file.FileReferenceService import org.springframework.data.domain.PageRequest import org.springframework.data.mongodb.core.query.Query import org.springframework.data.mongodb.core.query.and @@ -121,12 +121,12 @@ class DeletedNodeCleanupJob( if (!node.folder && (node.clusterNames == null || node.clusterNames!!.contains(clusterProperties.self.name)) ) { - fileReferenceChanged = fileReferenceService.decrement(node, repo) + fileReferenceChanged = fileReferenceService.decrement(node.sha256!!, repo.credentialsKey) } } catch (ignored: Exception) { logger.error("Clean up deleted node[$node] failed.", ignored) if (fileReferenceChanged) { - fileReferenceService.increment(node, repo) + fileReferenceService.increment(node.sha256!!, repo.credentialsKey) } } } diff --git a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/blocknode/BlockNodeServiceImpl.kt b/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/blocknode/BlockNodeServiceImpl.kt index bd8d5b9f31..417922e2e8 100644 --- a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/blocknode/BlockNodeServiceImpl.kt +++ b/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/blocknode/BlockNodeServiceImpl.kt @@ -28,11 +28,11 @@ package com.tencent.bkrepo.repository.service.blocknode import com.tencent.bkrepo.common.artifact.exception.NodeNotFoundException -import com.tencent.bkrepo.common.metadata.dao.BlockNodeDao +import com.tencent.bkrepo.common.metadata.dao.blocknode.BlockNodeDao import com.tencent.bkrepo.common.metadata.service.blocknode.impl.AbstractBlockNodeService import com.tencent.bkrepo.repository.dao.NodeDao import com.tencent.bkrepo.repository.pojo.node.NodeDetail -import com.tencent.bkrepo.repository.service.file.FileReferenceService +import com.tencent.bkrepo.common.metadata.service.file.FileReferenceService import com.tencent.bkrepo.repository.service.node.impl.NodeBaseService import org.springframework.context.annotation.Primary import org.springframework.stereotype.Service @@ -41,13 +41,9 @@ import org.springframework.stereotype.Service @Primary class BlockNodeServiceImpl( blockNodeDao: BlockNodeDao, - private val fileReferenceService: FileReferenceService, + fileReferenceService: FileReferenceService, private val nodeDao: NodeDao -) : AbstractBlockNodeService(blockNodeDao) { - - override fun incFileRef(sha256: String, credentialsKey: String?) { - fileReferenceService.increment(sha256, credentialsKey) - } +) : AbstractBlockNodeService(blockNodeDao, fileReferenceService) { override fun getNodeDetail(projectId: String, repoName: String, fullPath: String): NodeDetail { val node = nodeDao.findNode(projectId, repoName, fullPath) ?: throw NodeNotFoundException(fullPath) diff --git a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/impl/NodeBaseService.kt b/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/impl/NodeBaseService.kt index 6758011107..e752d964bc 100644 --- a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/impl/NodeBaseService.kt +++ b/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/impl/NodeBaseService.kt @@ -68,7 +68,7 @@ import com.tencent.bkrepo.repository.pojo.node.service.NodeCreateRequest import com.tencent.bkrepo.repository.pojo.node.service.NodeLinkRequest import com.tencent.bkrepo.repository.pojo.node.service.NodeUpdateAccessDateRequest import com.tencent.bkrepo.repository.pojo.node.service.NodeUpdateRequest -import com.tencent.bkrepo.repository.service.file.FileReferenceService +import com.tencent.bkrepo.common.metadata.service.file.FileReferenceService import com.tencent.bkrepo.repository.service.node.NodeService import com.tencent.bkrepo.repository.service.repo.QuotaService import com.tencent.bkrepo.repository.service.repo.StorageCredentialService @@ -344,7 +344,7 @@ abstract class NodeBaseService( if (!node.folder) { // 软链接node或fs-server创建的node的sha256为FAKE_SHA256不会关联实际文件，无需增加引用数 if (node.sha256 != FAKE_SHA256) { - fileReferenceService.increment(node, repository) + incrementFileReference(node, repository) } quotaService.increaseUsedVolume(node.projectId, node.repoName, node.size) } @@ -423,6 +423,35 @@ abstract class NodeBaseService( ) } + private fun incrementFileReference(node: TNode, repository: TRepository?): Boolean { + if (!validateParameter(node)) return false + return try { + val credentialsKey = findCredentialsKey(node, repository) + fileReferenceService.increment(node.sha256!!, credentialsKey) + } catch (exception: IllegalArgumentException) { + logger.error("Failed to increment reference of node [$node], repository not found.") + false + } + } + + private fun findCredentialsKey(node: TNode, repository: TRepository?): String? { + if (repository != null) { + return repository.credentialsKey + } + val tRepository = repositoryDao.findByNameAndType(node.projectId, node.repoName) + require(tRepository != null) + return tRepository.credentialsKey + } + + private fun validateParameter(node: TNode): Boolean { + if (node.folder) return false + if (node.sha256.isNullOrBlank()) { + logger.warn("Failed to change file reference, node[$node] sha256 is null or blank.") + return false + } + return true + } + /** * 获取用户无权限路径列表 */ diff --git a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/impl/NodeServiceImpl.kt b/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/impl/NodeServiceImpl.kt index 86bc0a7987..34bd563c4d 100644 --- a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/impl/NodeServiceImpl.kt +++ b/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/impl/NodeServiceImpl.kt @@ -52,7 +52,7 @@ import com.tencent.bkrepo.repository.pojo.node.service.NodeMoveCopyRequest import com.tencent.bkrepo.repository.pojo.node.service.NodeRenameRequest import com.tencent.bkrepo.repository.pojo.node.service.NodeUnCompressedRequest import com.tencent.bkrepo.repository.pojo.node.service.NodesDeleteRequest -import com.tencent.bkrepo.repository.service.file.FileReferenceService +import com.tencent.bkrepo.common.metadata.service.file.FileReferenceService import com.tencent.bkrepo.repository.service.repo.QuotaService import com.tencent.bkrepo.repository.service.repo.StorageCredentialService import com.tencent.bkrepo.router.api.RouterControllerClient diff --git a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/impl/center/CommitEdgeCenterNodeServiceImpl.kt b/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/impl/center/CommitEdgeCenterNodeServiceImpl.kt index 3ded79ab00..80785959e3 100644 --- a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/impl/center/CommitEdgeCenterNodeServiceImpl.kt +++ b/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/impl/center/CommitEdgeCenterNodeServiceImpl.kt @@ -56,7 +56,7 @@ import com.tencent.bkrepo.repository.pojo.node.service.NodeDeleteRequest import com.tencent.bkrepo.repository.pojo.node.service.NodeMoveCopyRequest import com.tencent.bkrepo.repository.pojo.node.service.NodeRenameRequest import com.tencent.bkrepo.repository.pojo.node.service.NodesDeleteRequest -import com.tencent.bkrepo.repository.service.file.FileReferenceService +import com.tencent.bkrepo.common.metadata.service.file.FileReferenceService import com.tencent.bkrepo.repository.service.node.impl.NodeRestoreSupport import com.tencent.bkrepo.repository.service.node.impl.NodeServiceImpl import com.tencent.bkrepo.repository.service.repo.QuotaService diff --git a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/impl/edge/EdgeNodeBaseService.kt b/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/impl/edge/EdgeNodeBaseService.kt index bcc9388a07..083e442d95 100644 --- a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/impl/edge/EdgeNodeBaseService.kt +++ b/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/impl/edge/EdgeNodeBaseService.kt @@ -44,7 +44,7 @@ import com.tencent.bkrepo.repository.pojo.node.NodeDetail import com.tencent.bkrepo.repository.pojo.node.service.NodeCreateRequest import com.tencent.bkrepo.repository.pojo.node.service.NodeUpdateAccessDateRequest import com.tencent.bkrepo.repository.pojo.node.service.NodeUpdateRequest -import com.tencent.bkrepo.repository.service.file.FileReferenceService +import com.tencent.bkrepo.common.metadata.service.file.FileReferenceService import com.tencent.bkrepo.repository.service.node.impl.NodeBaseService import com.tencent.bkrepo.repository.service.repo.QuotaService import com.tencent.bkrepo.repository.service.repo.StorageCredentialService diff --git a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/impl/edge/EdgeNodeServiceImpl.kt b/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/impl/edge/EdgeNodeServiceImpl.kt index cf723ba665..da8ea2361b 100644 --- a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/impl/edge/EdgeNodeServiceImpl.kt +++ b/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/impl/edge/EdgeNodeServiceImpl.kt @@ -57,7 +57,7 @@ import com.tencent.bkrepo.repository.pojo.node.service.NodeRenameRequest import com.tencent.bkrepo.repository.pojo.node.service.NodeRestoreRequest import com.tencent.bkrepo.repository.pojo.node.service.NodeUnCompressedRequest import com.tencent.bkrepo.repository.pojo.node.service.NodesDeleteRequest -import com.tencent.bkrepo.repository.service.file.FileReferenceService +import com.tencent.bkrepo.common.metadata.service.file.FileReferenceService import com.tencent.bkrepo.repository.service.node.impl.NodeArchiveSupport import com.tencent.bkrepo.repository.service.node.impl.NodeCompressSupport import com.tencent.bkrepo.repository.service.node.impl.NodeDeleteSupport diff --git a/src/backend/repository/biz-repository/src/test/kotlin/com/tencent/bkrepo/repository/service/FileReferenceServiceTest.kt b/src/backend/repository/biz-repository/src/test/kotlin/com/tencent/bkrepo/repository/service/FileReferenceServiceTest.kt index 8e5d6cb8d6..da7303a01d 100644 --- a/src/backend/repository/biz-repository/src/test/kotlin/com/tencent/bkrepo/repository/service/FileReferenceServiceTest.kt +++ b/src/backend/repository/biz-repository/src/test/kotlin/com/tencent/bkrepo/repository/service/FileReferenceServiceTest.kt @@ -38,11 +38,11 @@ import com.tencent.bkrepo.repository.UT_PROJECT_ID import com.tencent.bkrepo.repository.UT_REPO_NAME import com.tencent.bkrepo.repository.UT_STORAGE_CREDENTIALS_KEY import com.tencent.bkrepo.repository.UT_USER -import com.tencent.bkrepo.repository.dao.FileReferenceDao +import com.tencent.bkrepo.common.metadata.dao.file.FileReferenceDao import com.tencent.bkrepo.repository.dao.NodeDao import com.tencent.bkrepo.repository.pojo.credendials.StorageCredentialsCreateRequest import com.tencent.bkrepo.repository.pojo.node.service.NodeCreateRequest -import com.tencent.bkrepo.repository.service.file.FileReferenceService +import com.tencent.bkrepo.common.metadata.service.file.FileReferenceService import com.tencent.bkrepo.repository.service.node.NodeService import com.tencent.bkrepo.repository.service.repo.ProjectService import com.tencent.bkrepo.repository.service.repo.RepositoryService diff --git a/src/backend/repository/biz-repository/src/test/kotlin/com/tencent/bkrepo/repository/service/MetadataServiceTest.kt b/src/backend/repository/biz-repository/src/test/kotlin/com/tencent/bkrepo/repository/service/MetadataServiceTest.kt index ed9af35e22..f72386165d 100644 --- a/src/backend/repository/biz-repository/src/test/kotlin/com/tencent/bkrepo/repository/service/MetadataServiceTest.kt +++ b/src/backend/repository/biz-repository/src/test/kotlin/com/tencent/bkrepo/repository/service/MetadataServiceTest.kt @@ -35,7 +35,7 @@ import com.tencent.bkrepo.common.artifact.path.PathUtils.ROOT import com.tencent.bkrepo.repository.UT_PROJECT_ID import com.tencent.bkrepo.repository.UT_REPO_NAME import com.tencent.bkrepo.repository.UT_USER -import com.tencent.bkrepo.repository.dao.FileReferenceDao +import com.tencent.bkrepo.common.metadata.dao.file.FileReferenceDao import com.tencent.bkrepo.repository.dao.NodeDao import com.tencent.bkrepo.repository.pojo.metadata.MetadataDeleteRequest import com.tencent.bkrepo.repository.pojo.metadata.MetadataModel diff --git a/src/backend/repository/biz-repository/src/test/kotlin/com/tencent/bkrepo/repository/service/NodeQueryWithoutShardingKeyTest.kt b/src/backend/repository/biz-repository/src/test/kotlin/com/tencent/bkrepo/repository/service/NodeQueryWithoutShardingKeyTest.kt index 4a1bd7efbf..f9afce3eb0 100644 --- a/src/backend/repository/biz-repository/src/test/kotlin/com/tencent/bkrepo/repository/service/NodeQueryWithoutShardingKeyTest.kt +++ b/src/backend/repository/biz-repository/src/test/kotlin/com/tencent/bkrepo/repository/service/NodeQueryWithoutShardingKeyTest.kt @@ -31,7 +31,7 @@ import com.tencent.bkrepo.repository.UT_PROJECT_ID import com.tencent.bkrepo.repository.UT_REPO_NAME import com.tencent.bkrepo.repository.UT_USER import com.tencent.bkrepo.repository.constant.SHARDING_COUNT -import com.tencent.bkrepo.repository.dao.FileReferenceDao +import com.tencent.bkrepo.common.metadata.dao.file.FileReferenceDao import com.tencent.bkrepo.repository.dao.NodeDao import com.tencent.bkrepo.repository.pojo.metadata.MetadataModel import com.tencent.bkrepo.repository.pojo.node.NodeDetail diff --git a/src/backend/repository/biz-repository/src/test/kotlin/com/tencent/bkrepo/repository/service/NodeSearchServiceTest.kt b/src/backend/repository/biz-repository/src/test/kotlin/com/tencent/bkrepo/repository/service/NodeSearchServiceTest.kt index 0ffdd85f4b..5feb84ce15 100644 --- a/src/backend/repository/biz-repository/src/test/kotlin/com/tencent/bkrepo/repository/service/NodeSearchServiceTest.kt +++ b/src/backend/repository/biz-repository/src/test/kotlin/com/tencent/bkrepo/repository/service/NodeSearchServiceTest.kt @@ -43,7 +43,7 @@ import com.tencent.bkrepo.common.service.util.ResponseBuilder import com.tencent.bkrepo.repository.UT_PROJECT_ID import com.tencent.bkrepo.repository.UT_REPO_NAME import com.tencent.bkrepo.repository.UT_USER -import com.tencent.bkrepo.repository.dao.FileReferenceDao +import com.tencent.bkrepo.common.metadata.dao.file.FileReferenceDao import com.tencent.bkrepo.repository.dao.NodeDao import com.tencent.bkrepo.repository.pojo.metadata.MetadataModel import com.tencent.bkrepo.repository.pojo.node.NodeInfo diff --git a/src/backend/repository/biz-repository/src/test/kotlin/com/tencent/bkrepo/repository/service/NodeServiceTest.kt b/src/backend/repository/biz-repository/src/test/kotlin/com/tencent/bkrepo/repository/service/NodeServiceTest.kt index 81d071506d..1967535b2b 100644 --- a/src/backend/repository/biz-repository/src/test/kotlin/com/tencent/bkrepo/repository/service/NodeServiceTest.kt +++ b/src/backend/repository/biz-repository/src/test/kotlin/com/tencent/bkrepo/repository/service/NodeServiceTest.kt @@ -41,7 +41,7 @@ import com.tencent.bkrepo.common.service.util.ResponseBuilder import com.tencent.bkrepo.repository.UT_PROJECT_ID import com.tencent.bkrepo.repository.UT_REPO_NAME import com.tencent.bkrepo.repository.UT_USER -import com.tencent.bkrepo.repository.dao.FileReferenceDao +import com.tencent.bkrepo.common.metadata.dao.file.FileReferenceDao import com.tencent.bkrepo.repository.dao.NodeDao import com.tencent.bkrepo.repository.pojo.metadata.MetadataModel import com.tencent.bkrepo.repository.pojo.node.NodeListOption diff --git a/src/backend/repository/biz-repository/src/test/kotlin/com/tencent/bkrepo/repository/service/ServiceBaseTest.kt b/src/backend/repository/biz-repository/src/test/kotlin/com/tencent/bkrepo/repository/service/ServiceBaseTest.kt index 928b6455c7..652b7ec874 100644 --- a/src/backend/repository/biz-repository/src/test/kotlin/com/tencent/bkrepo/repository/service/ServiceBaseTest.kt +++ b/src/backend/repository/biz-repository/src/test/kotlin/com/tencent/bkrepo/repository/service/ServiceBaseTest.kt @@ -43,7 +43,7 @@ import com.tencent.bkrepo.common.artifact.pojo.RepositoryCategory import com.tencent.bkrepo.common.artifact.pojo.RepositoryType import com.tencent.bkrepo.common.artifact.pojo.configuration.local.LocalConfiguration import com.tencent.bkrepo.common.artifact.router.RouterControllerProperties -import com.tencent.bkrepo.common.metadata.dao.BlockNodeDao +import com.tencent.bkrepo.common.metadata.dao.blocknode.BlockNodeDao import com.tencent.bkrepo.common.security.http.core.HttpAuthProperties import com.tencent.bkrepo.common.security.manager.PermissionManager import com.tencent.bkrepo.common.security.manager.ci.CIPermissionManager @@ -100,7 +100,7 @@ import org.springframework.test.context.TestPropertySource RouterControllerProperties::class, BlockNodeDao::class ) -@ComponentScan("com.tencent.bkrepo.repository.service") +@ComponentScan(value = ["com.tencent.bkrepo.repository.service", "com.tencent.bkrepo.common.metadata"]) @TestPropertySource(locations = ["classpath:bootstrap-ut.properties", "classpath:center-ut.properties"]) open class ServiceBaseTest { diff --git a/src/backend/repository/biz-repository/src/test/kotlin/com/tencent/bkrepo/repository/service/StorageCredentialServiceTest.kt b/src/backend/repository/biz-repository/src/test/kotlin/com/tencent/bkrepo/repository/service/StorageCredentialServiceTest.kt index 73f42a97e3..36e3edef8e 100644 --- a/src/backend/repository/biz-repository/src/test/kotlin/com/tencent/bkrepo/repository/service/StorageCredentialServiceTest.kt +++ b/src/backend/repository/biz-repository/src/test/kotlin/com/tencent/bkrepo/repository/service/StorageCredentialServiceTest.kt @@ -40,7 +40,7 @@ import com.tencent.bkrepo.common.storage.credentials.StorageCredentials import com.tencent.bkrepo.repository.UT_REGION import com.tencent.bkrepo.repository.UT_STORAGE_CREDENTIALS_KEY import com.tencent.bkrepo.repository.UT_USER -import com.tencent.bkrepo.repository.dao.FileReferenceDao +import com.tencent.bkrepo.common.metadata.dao.file.FileReferenceDao import com.tencent.bkrepo.repository.dao.NodeDao import com.tencent.bkrepo.repository.dao.repository.StorageCredentialsRepository import com.tencent.bkrepo.repository.pojo.credendials.StorageCredentialsCreateRequest diff --git a/src/backend/repository/biz-repository/src/test/kotlin/com/tencent/bkrepo/repository/service/node/NodeRestoreSupportTest.kt b/src/backend/repository/biz-repository/src/test/kotlin/com/tencent/bkrepo/repository/service/node/NodeRestoreSupportTest.kt index 4c47d8d205..65bdbfd990 100644 --- a/src/backend/repository/biz-repository/src/test/kotlin/com/tencent/bkrepo/repository/service/node/NodeRestoreSupportTest.kt +++ b/src/backend/repository/biz-repository/src/test/kotlin/com/tencent/bkrepo/repository/service/node/NodeRestoreSupportTest.kt @@ -37,7 +37,7 @@ import com.tencent.bkrepo.common.artifact.api.DefaultArtifactInfo import com.tencent.bkrepo.repository.UT_PROJECT_ID import com.tencent.bkrepo.repository.UT_REPO_NAME import com.tencent.bkrepo.repository.constant.SYSTEM_USER -import com.tencent.bkrepo.repository.dao.FileReferenceDao +import com.tencent.bkrepo.common.metadata.dao.file.FileReferenceDao import com.tencent.bkrepo.repository.dao.NodeDao import com.tencent.bkrepo.repository.model.TNode import com.tencent.bkrepo.repository.pojo.node.ConflictStrategy