diff --git a/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/repository/remote/RemoteRepository.kt b/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/repository/remote/RemoteRepository.kt index 7f9e3ffd5f..9fd45442c0 100644 --- a/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/repository/remote/RemoteRepository.kt +++ b/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/repository/remote/RemoteRepository.kt @@ -115,21 +115,19 @@ abstract class RemoteRepository : AbstractArtifactRepository() { /** * 尝试读取缓存的远程构件 */ - fun getCacheArtifactResource(context: ArtifactDownloadContext): ArtifactResource? { + open fun getCacheArtifactResource(context: ArtifactContext): ArtifactResource? { val configuration = context.getRemoteConfiguration() if (!configuration.cache.enabled) return null val cacheNode = findCacheNodeDetail(context) - if (cacheNode == null || cacheNode.folder) return null - return if (!isExpired(cacheNode, configuration.cache.expiration)) { - loadArtifactResource(cacheNode, context) - } else null + return if (cacheNode == null || cacheNode.folder || isExpired(cacheNode, configuration.cache.expiration)) null + else loadArtifactResource(cacheNode, context) } /** * 加载要返回的资源 */ - open fun loadArtifactResource(cacheNode: NodeDetail, context: ArtifactDownloadContext): ArtifactResource? { + open fun loadArtifactResource(cacheNode: NodeDetail, context: ArtifactContext): ArtifactResource? { return storageService.load(cacheNode.sha256!!, Range.full(cacheNode.size), context.storageCredentials)?.run { if (logger.isDebugEnabled) { logger.debug("Cached remote artifact[${context.artifactInfo}] is hit.") @@ -152,7 +150,7 @@ abstract class RemoteRepository : AbstractArtifactRepository() { /** * 尝试获取缓存的远程构件节点 */ - open fun findCacheNodeDetail(context: ArtifactDownloadContext): NodeDetail? { + open fun findCacheNodeDetail(context: ArtifactContext): NodeDetail? { with(context) { return nodeClient.getNodeDetail(projectId, repoName, artifactInfo.getArtifactFullPath()).data } @@ -177,7 +175,7 @@ abstract class RemoteRepository : AbstractArtifactRepository() { val size = artifactFile.getSize() val artifactStream = artifactFile.getInputStream().artifactStream(Range.full(size)) val node = cacheArtifactFile(context, artifactFile) - return ArtifactResource(artifactStream, context.artifactInfo.getResponseName(), node, ArtifactChannel.LOCAL) + return ArtifactResource(artifactStream, context.artifactInfo.getResponseName(), node, ArtifactChannel.PROXY) } /** diff --git a/src/backend/npm/api-npm/src/main/kotlin/com/tencent/bkrepo/npm/constants/Constants.kt b/src/backend/npm/api-npm/src/main/kotlin/com/tencent/bkrepo/npm/constants/Constants.kt index 592b61a499..0ed2e6ab65 100644 --- a/src/backend/npm/api-npm/src/main/kotlin/com/tencent/bkrepo/npm/constants/Constants.kt +++ b/src/backend/npm/api-npm/src/main/kotlin/com/tencent/bkrepo/npm/constants/Constants.kt @@ -62,10 +62,10 @@ const val NPM_PKG_VERSION_METADATA_FULL_PATH = "/.npm/%s/%s-%s.json" const val NPM_PKG_METADATA_FULL_PATH = "/.npm/%s/package.json" const val NPM_FILE_FULL_PATH = "npm_file_full_path" - const val SEARCH_REQUEST = "search_request" - const val PKG_NAME = "pkg_name" +const val REQUEST_URI = "requestURI" +const val PACKAGE_JSON = "package.json" // constants map val ERROR_MAP = mapOf("error" to "not_found", "reason" to "document not found") diff --git a/src/backend/npm/biz-npm/src/main/kotlin/com/tencent/bkrepo/npm/artifact/repository/NpmRemoteRepository.kt b/src/backend/npm/biz-npm/src/main/kotlin/com/tencent/bkrepo/npm/artifact/repository/NpmRemoteRepository.kt index 7ae6350c04..561a8e413f 100644 --- a/src/backend/npm/biz-npm/src/main/kotlin/com/tencent/bkrepo/npm/artifact/repository/NpmRemoteRepository.kt +++ b/src/backend/npm/biz-npm/src/main/kotlin/com/tencent/bkrepo/npm/artifact/repository/NpmRemoteRepository.kt @@ -31,6 +31,7 @@ package com.tencent.bkrepo.npm.artifact.repository +import com.tencent.bkrepo.common.api.constant.MediaTypes.APPLICATION_JSON_WITHOUT_CHARSET import com.tencent.bkrepo.common.api.util.JsonUtils import com.tencent.bkrepo.common.artifact.api.ArtifactFile import com.tencent.bkrepo.common.artifact.repository.context.ArtifactContext @@ -45,12 +46,14 @@ import com.tencent.bkrepo.common.artifact.resolve.response.ArtifactResource import com.tencent.bkrepo.common.artifact.util.http.UrlFormatter import com.tencent.bkrepo.common.storage.monitor.Throughput import com.tencent.bkrepo.npm.constants.NPM_FILE_FULL_PATH +import com.tencent.bkrepo.npm.constants.PACKAGE_JSON +import com.tencent.bkrepo.npm.constants.REQUEST_URI import com.tencent.bkrepo.npm.exception.NpmBadRequestException import com.tencent.bkrepo.npm.pojo.NpmSearchInfoMap import com.tencent.bkrepo.npm.pojo.NpmSearchResponse import com.tencent.bkrepo.npm.utils.NpmUtils +import com.tencent.bkrepo.repository.pojo.node.NodeDetail import com.tencent.bkrepo.repository.pojo.node.service.NodeCreateRequest -import okhttp3.Request import okhttp3.Response import org.slf4j.Logger import org.slf4j.LoggerFactory @@ -69,34 +72,13 @@ class NpmRemoteRepository( throughput: Throughput ) { super.onDownloadSuccess(context, artifactResource, throughput) + val packageInfo = NpmUtils.parseNameAndVersionFromFullPath(context.artifactInfo.getArtifactFullPath()) + val versionMetadataFullPath = NpmUtils.getVersionPackageMetadataPath(packageInfo.first, packageInfo.second) + val queryContext = ArtifactQueryContext(context.repo, context.artifactInfo) + queryContext.putAttribute(NPM_FILE_FULL_PATH, versionMetadataFullPath) + queryContext.putAttribute(REQUEST_URI, "/${packageInfo.first}/${packageInfo.second}") // 存储package-version.json文件 - executor.execute { cachePackageVersionMetadata(context) } - } - - private fun cachePackageVersionMetadata(context: ArtifactDownloadContext) { - with(context) { - val packageInfo = NpmUtils.parseNameAndVersionFromFullPath(artifactInfo.getArtifactFullPath()) - val versionMetadataFullPath = NpmUtils.getVersionPackageMetadataPath(packageInfo.first, packageInfo.second) - if (nodeClient.checkExist(projectId, repoName, versionMetadataFullPath).data!!) { - logger.info( - "version metadata [$versionMetadataFullPath] is already exits " + - "in repo [$projectId/$repoName]" - ) - return - } - val remoteConfiguration = context.getRemoteConfiguration() - val httpClient = createHttpClient(remoteConfiguration) - context.putAttribute("requestURI", "/${packageInfo.first}/${packageInfo.second}") - val downloadUri = createRemoteSearchUrl(context) - val request = Request.Builder().url(downloadUri).build() - val response = httpClient.newCall(request).execute() - if (checkResponse(response)) { - val artifactFile = createTempFile(response.body!!) - context.putAttribute(NPM_FILE_FULL_PATH, versionMetadataFullPath) - cacheArtifactFile(context, artifactFile) - logger.info("cache version metadata [$versionMetadataFullPath] success.") - } - } + executor.execute { findCacheNodeDetail(queryContext) ?: (super.query(queryContext) as InputStream?)?.close() } } override fun upload(context: ArtifactUploadContext) { @@ -108,19 +90,44 @@ class NpmRemoteRepository( } override fun query(context: ArtifactQueryContext): InputStream? { - val remoteConfiguration = context.getRemoteConfiguration() - val httpClient = createHttpClient(remoteConfiguration) - val downloadUri = createRemoteSearchUrl(context) - val request = Request.Builder().url(downloadUri).build() - val response = httpClient.newCall(request).execute() - return if (checkResponse(response)) { - onQueryResponse(context, response) - } else null + return getCacheArtifactResource(context)?.getSingleStream() ?: super.query(context) as InputStream? } - private fun createRemoteSearchUrl(context: ArtifactContext): String { + override fun checkQueryResponse(response: Response): Boolean { + return super.checkQueryResponse(response) && run { + val contentType = response.body!!.contentType() + contentType.toString().contains(APPLICATION_JSON_WITHOUT_CHARSET) || run { + logger.warn("Content-Type($contentType) of response from [${response.request.url}] is unsupported") + false + } + } + } + + // 仅package.json文件有必要在缓存过期后更新 + override fun getCacheArtifactResource(context: ArtifactContext): ArtifactResource? { + return when (context) { + is ArtifactDownloadContext -> findCacheNodeDetail(context)?.let { loadArtifactResource(it, context) } + is ArtifactQueryContext -> { + if (context.getStringAttribute(NPM_FILE_FULL_PATH)?.endsWith("/$PACKAGE_JSON") == false) { + findCacheNodeDetail(context)?.let { loadArtifactResource(it, context) } + } else if (context.getRemoteConfiguration().cache.expiration > 0) { + super.getCacheArtifactResource(context) + } else null + } + else -> null + } + } + + override fun findCacheNodeDetail(context: ArtifactContext): NodeDetail? { + val fullPath = context.getStringAttribute(NPM_FILE_FULL_PATH)!! + with(context) { + return nodeClient.getNodeDetail(projectId, repoName, fullPath).data + } + } + + override fun createRemoteDownloadUrl(context: ArtifactContext): String { val configuration = context.getRemoteConfiguration() - val requestURI = context.getStringAttribute("requestURI") + val requestURI = context.getStringAttribute(REQUEST_URI) val artifactUri = requestURI ?: context.request.requestURI.substringAfterLast(context.artifactInfo.getRepoIdentify()) val queryString = context.request.queryString @@ -128,22 +135,9 @@ class NpmRemoteRepository( } override fun onQueryResponse(context: ArtifactQueryContext, response: Response): InputStream? { - val fullPath = context.getStringAttribute(NPM_FILE_FULL_PATH)!! val body = response.body!! val artifactFile = createTempFile(body) - val sha256 = artifactFile.getFileSha256() - with(context) { - nodeClient.getNodeDetail(projectId, repoName, fullPath).data?.let { - if (it.sha256.equals(sha256)) { - logger.info("artifact [$fullPath] is hit the cache.") - return artifactFile.getInputStream() - } - cacheArtifactFile(context, artifactFile) - } ?: run { - // 存储构件 - cacheArtifactFile(context, artifactFile) - } - } + cacheArtifactFile(context, artifactFile) return artifactFile.getInputStream() } diff --git a/src/backend/npm/biz-npm/src/main/kotlin/com/tencent/bkrepo/npm/service/impl/NpmClientServiceImpl.kt b/src/backend/npm/biz-npm/src/main/kotlin/com/tencent/bkrepo/npm/service/impl/NpmClientServiceImpl.kt index 67e7049042..c190188300 100644 --- a/src/backend/npm/biz-npm/src/main/kotlin/com/tencent/bkrepo/npm/service/impl/NpmClientServiceImpl.kt +++ b/src/backend/npm/biz-npm/src/main/kotlin/com/tencent/bkrepo/npm/service/impl/NpmClientServiceImpl.kt @@ -53,6 +53,7 @@ import com.tencent.bkrepo.npm.constants.LATEST import com.tencent.bkrepo.npm.constants.MODIFIED import com.tencent.bkrepo.npm.constants.NPM_FILE_FULL_PATH import com.tencent.bkrepo.npm.constants.NPM_PACKAGE_TGZ_FILE +import com.tencent.bkrepo.npm.constants.REQUEST_URI import com.tencent.bkrepo.npm.constants.SEARCH_REQUEST import com.tencent.bkrepo.npm.constants.SIZE import com.tencent.bkrepo.npm.exception.NpmArtifactExistException @@ -275,6 +276,7 @@ class NpmClientServiceImpl( val context = ArtifactQueryContext() val packageFullPath = NpmUtils.getPackageMetadataPath(name) context.putAttribute(NPM_FILE_FULL_PATH, packageFullPath) + context.putAttribute(REQUEST_URI, name) val inputStream = ArtifactContextHolder.getRepository().query(context) as? InputStream ?: throw NpmArtifactNotFoundException("document not found") @@ -304,6 +306,7 @@ class NpmClientServiceImpl( val context = ArtifactQueryContext() val packageFullPath = NpmUtils.getVersionPackageMetadataPath(name, version) context.putAttribute(NPM_FILE_FULL_PATH, packageFullPath) + context.putAttribute(REQUEST_URI, "$name/$version") val inputStream = ArtifactContextHolder.getRepository().query(context) as? InputStream ?: throw NpmArtifactNotFoundException("document not found") diff --git a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/artifact/repository/OciRegistryRemoteRepository.kt b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/artifact/repository/OciRegistryRemoteRepository.kt index 8a6833e65b..3fcf67ee62 100644 --- a/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/artifact/repository/OciRegistryRemoteRepository.kt +++ b/src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/artifact/repository/OciRegistryRemoteRepository.kt @@ -453,7 +453,7 @@ class OciRegistryRemoteRepository( /** * 尝试获取缓存的远程构件节点 */ - override fun findCacheNodeDetail(context: ArtifactDownloadContext): NodeDetail? { + override fun findCacheNodeDetail(context: ArtifactContext): NodeDetail? { with(context) { val fullPath = ociOperationService.getNodeFullPath(context.artifactInfo as OciArtifactInfo) ?: return null return nodeClient.getNodeDetail(projectId, repoName, fullPath).data @@ -463,7 +463,7 @@ class OciRegistryRemoteRepository( /** * 加载要返回的资源: oci协议需要返回特定的请求头和资源类型 */ - override fun loadArtifactResource(cacheNode: NodeDetail, context: ArtifactDownloadContext): ArtifactResource? { + override fun loadArtifactResource(cacheNode: NodeDetail, context: ArtifactContext): ArtifactResource? { return storageService.load(cacheNode.sha256!!, Range.full(cacheNode.size), context.storageCredentials)?.run { if (logger.isDebugEnabled) { logger.debug("Cached remote artifact[${context.artifactInfo}] is hit.") @@ -484,7 +484,7 @@ class OciRegistryRemoteRepository( private fun buildResponse( cacheNode: NodeDetail?, - context: ArtifactDownloadContext, + context: ArtifactContext, artifactResource: ArtifactResource, sha256: String? = null, size: Long? = null