refactor: npm远程仓库优化 #1168

src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/repository/remote/RemoteRepository.kt

@@ -115,21 +115,19 @@ abstract class RemoteRepository : AbstractArtifactRepository() {
     /**
      * 尝试读取缓存的远程构件
      */
-    fun getCacheArtifactResource(context: ArtifactDownloadContext): ArtifactResource? {
+    open fun getCacheArtifactResource(context: ArtifactContext): ArtifactResource? {
         val configuration = context.getRemoteConfiguration()
         if (!configuration.cache.enabled) return null
 
         val cacheNode = findCacheNodeDetail(context)
-        if (cacheNode == null || cacheNode.folder) return null
-        return if (!isExpired(cacheNode, configuration.cache.expiration)) {
-            loadArtifactResource(cacheNode, context)
-        } else null
+        return if (cacheNode == null || cacheNode.folder || isExpired(cacheNode, configuration.cache.expiration)) null
+        else loadArtifactResource(cacheNode, context)
     }
 
     /**
      * 加载要返回的资源
      */
-    open fun loadArtifactResource(cacheNode: NodeDetail, context: ArtifactDownloadContext): ArtifactResource? {
+    open fun loadArtifactResource(cacheNode: NodeDetail, context: ArtifactContext): ArtifactResource? {
         return storageService.load(cacheNode.sha256!!, Range.full(cacheNode.size), context.storageCredentials)?.run {
             if (logger.isDebugEnabled) {
                 logger.debug("Cached remote artifact[${context.artifactInfo}] is hit.")
@@ -152,7 +150,7 @@ abstract class RemoteRepository : AbstractArtifactRepository() {
     /**
      * 尝试获取缓存的远程构件节点
      */
-    open fun findCacheNodeDetail(context: ArtifactDownloadContext): NodeDetail? {
+    open fun findCacheNodeDetail(context: ArtifactContext): NodeDetail? {
         with(context) {
             return nodeClient.getNodeDetail(projectId, repoName, artifactInfo.getArtifactFullPath()).data
         }
@@ -177,7 +175,7 @@ abstract class RemoteRepository : AbstractArtifactRepository() {
         val size = artifactFile.getSize()
         val artifactStream = artifactFile.getInputStream().artifactStream(Range.full(size))
         val node = cacheArtifactFile(context, artifactFile)
-        return ArtifactResource(artifactStream, context.artifactInfo.getResponseName(), node, ArtifactChannel.LOCAL)
+        return ArtifactResource(artifactStream, context.artifactInfo.getResponseName(), node, ArtifactChannel.PROXY)
     }
 
     /**

src/backend/npm/api-npm/src/main/kotlin/com/tencent/bkrepo/npm/constants/Constants.kt

@@ -62,10 +62,10 @@ const val NPM_PKG_VERSION_METADATA_FULL_PATH = "/.npm/%s/%s-%s.json"
 const val NPM_PKG_METADATA_FULL_PATH = "/.npm/%s/package.json"
 
 const val NPM_FILE_FULL_PATH = "npm_file_full_path"
-
 const val SEARCH_REQUEST = "search_request"
-
 const val PKG_NAME = "pkg_name"
+const val REQUEST_URI = "requestURI"
+const val PACKAGE_JSON = "package.json"
 
 // constants map
 val ERROR_MAP = mapOf("error" to "not_found", "reason" to "document not found")

src/backend/npm/biz-npm/src/main/kotlin/com/tencent/bkrepo/npm/artifact/repository/NpmRemoteRepository.kt

@@ -31,6 +31,7 @@
 
 package com.tencent.bkrepo.npm.artifact.repository
 
+import com.tencent.bkrepo.common.api.constant.MediaTypes.APPLICATION_JSON_WITHOUT_CHARSET
 import com.tencent.bkrepo.common.api.util.JsonUtils
 import com.tencent.bkrepo.common.artifact.api.ArtifactFile
 import com.tencent.bkrepo.common.artifact.repository.context.ArtifactContext
@@ -45,12 +46,14 @@ import com.tencent.bkrepo.common.artifact.resolve.response.ArtifactResource
 import com.tencent.bkrepo.common.artifact.util.http.UrlFormatter
 import com.tencent.bkrepo.common.storage.monitor.Throughput
 import com.tencent.bkrepo.npm.constants.NPM_FILE_FULL_PATH
+import com.tencent.bkrepo.npm.constants.PACKAGE_JSON
+import com.tencent.bkrepo.npm.constants.REQUEST_URI
 import com.tencent.bkrepo.npm.exception.NpmBadRequestException
 import com.tencent.bkrepo.npm.pojo.NpmSearchInfoMap
 import com.tencent.bkrepo.npm.pojo.NpmSearchResponse
 import com.tencent.bkrepo.npm.utils.NpmUtils
+import com.tencent.bkrepo.repository.pojo.node.NodeDetail
 import com.tencent.bkrepo.repository.pojo.node.service.NodeCreateRequest
-import okhttp3.Request
 import okhttp3.Response
 import org.slf4j.Logger
 import org.slf4j.LoggerFactory
@@ -69,34 +72,13 @@ class NpmRemoteRepository(
         throughput: Throughput
     ) {
         super.onDownloadSuccess(context, artifactResource, throughput)
+        val packageInfo = NpmUtils.parseNameAndVersionFromFullPath(context.artifactInfo.getArtifactFullPath())
+        val versionMetadataFullPath = NpmUtils.getVersionPackageMetadataPath(packageInfo.first, packageInfo.second)
+        val queryContext = ArtifactQueryContext(context.repo, context.artifactInfo)
+        queryContext.putAttribute(NPM_FILE_FULL_PATH, versionMetadataFullPath)
+        queryContext.putAttribute(REQUEST_URI, "/${packageInfo.first}/${packageInfo.second}")
         // 存储package-version.json文件
-        executor.execute { cachePackageVersionMetadata(context) }
-    }
-
-    private fun cachePackageVersionMetadata(context: ArtifactDownloadContext) {
-        with(context) {
-            val packageInfo = NpmUtils.parseNameAndVersionFromFullPath(artifactInfo.getArtifactFullPath())
-            val versionMetadataFullPath = NpmUtils.getVersionPackageMetadataPath(packageInfo.first, packageInfo.second)
-            if (nodeClient.checkExist(projectId, repoName, versionMetadataFullPath).data!!) {
-                logger.info(
-                    "version metadata [$versionMetadataFullPath] is already exits " +
-                        "in repo [$projectId/$repoName]"
-                )
-                return
-            }
-            val remoteConfiguration = context.getRemoteConfiguration()
-            val httpClient = createHttpClient(remoteConfiguration)
-            context.putAttribute("requestURI", "/${packageInfo.first}/${packageInfo.second}")
-            val downloadUri = createRemoteSearchUrl(context)
-            val request = Request.Builder().url(downloadUri).build()
-            val response = httpClient.newCall(request).execute()
-            if (checkResponse(response)) {
-                val artifactFile = createTempFile(response.body!!)
-                context.putAttribute(NPM_FILE_FULL_PATH, versionMetadataFullPath)
-                cacheArtifactFile(context, artifactFile)
-                logger.info("cache version metadata [$versionMetadataFullPath] success.")
-            }
-        }
+        executor.execute { findCacheNodeDetail(queryContext) ?: (super.query(queryContext) as InputStream?)?.close() }
     }
 
     override fun upload(context: ArtifactUploadContext) {
@@ -108,42 +90,54 @@ class NpmRemoteRepository(
     }
 
     override fun query(context: ArtifactQueryContext): InputStream? {
-        val remoteConfiguration = context.getRemoteConfiguration()
-        val httpClient = createHttpClient(remoteConfiguration)
-        val downloadUri = createRemoteSearchUrl(context)
-        val request = Request.Builder().url(downloadUri).build()
-        val response = httpClient.newCall(request).execute()
-        return if (checkResponse(response)) {
-            onQueryResponse(context, response)
-        } else null
+        return getCacheArtifactResource(context)?.getSingleStream() ?: super.query(context) as InputStream?
     }
 
-    private fun createRemoteSearchUrl(context: ArtifactContext): String {
+    override fun checkQueryResponse(response: Response): Boolean {
+        return super.checkQueryResponse(response) && run {
+            val contentType = response.body!!.contentType()
+            contentType.toString().contains(APPLICATION_JSON_WITHOUT_CHARSET) || run {
+                logger.warn("Content-Type($contentType) of response from [${response.request.url}] is unsupported")
+                false
+            }
+        }
+    }
+
+    // 仅package.json文件有必要在缓存过期后更新
+    override fun getCacheArtifactResource(context: ArtifactContext): ArtifactResource? {
+        return when (context) {
+            is ArtifactDownloadContext -> findCacheNodeDetail(context)?.let { loadArtifactResource(it, context) }
+            is ArtifactQueryContext -> {
+                if (context.getStringAttribute(NPM_FILE_FULL_PATH)?.endsWith("/$PACKAGE_JSON") == false) {
+                    findCacheNodeDetail(context)?.let { loadArtifactResource(it, context) }
+                } else if (context.getRemoteConfiguration().cache.expiration > 0) {
+                    super.getCacheArtifactResource(context)
+                } else null
+            }
+            else -> null
+        }
+    }
+
+    override fun findCacheNodeDetail(context: ArtifactContext): NodeDetail? {
+        val fullPath = context.getStringAttribute(NPM_FILE_FULL_PATH)!!
+        with(context) {
+            return nodeClient.getNodeDetail(projectId, repoName, fullPath).data
+        }
+    }
+
+    override fun createRemoteDownloadUrl(context: ArtifactContext): String {
         val configuration = context.getRemoteConfiguration()
-        val requestURI = context.getStringAttribute("requestURI")
+        val requestURI = context.getStringAttribute(REQUEST_URI)
         val artifactUri =
             requestURI ?: context.request.requestURI.substringAfterLast(context.artifactInfo.getRepoIdentify())
         val queryString = context.request.queryString
         return UrlFormatter.format(configuration.url, artifactUri, queryString)
     }
 
     override fun onQueryResponse(context: ArtifactQueryContext, response: Response): InputStream? {
-        val fullPath = context.getStringAttribute(NPM_FILE_FULL_PATH)!!
         val body = response.body!!
         val artifactFile = createTempFile(body)
-        val sha256 = artifactFile.getFileSha256()
-        with(context) {
-            nodeClient.getNodeDetail(projectId, repoName, fullPath).data?.let {
-                if (it.sha256.equals(sha256)) {
-                    logger.info("artifact [$fullPath] is hit the cache.")
-                    return artifactFile.getInputStream()
-                }
-                cacheArtifactFile(context, artifactFile)
-            } ?: run {
-                // 存储构件
-                cacheArtifactFile(context, artifactFile)
-            }
-        }
+        cacheArtifactFile(context, artifactFile)
         return artifactFile.getInputStream()
     }
 

src/backend/npm/biz-npm/src/main/kotlin/com/tencent/bkrepo/npm/service/impl/NpmClientServiceImpl.kt

@@ -53,6 +53,7 @@ import com.tencent.bkrepo.npm.constants.LATEST
 import com.tencent.bkrepo.npm.constants.MODIFIED
 import com.tencent.bkrepo.npm.constants.NPM_FILE_FULL_PATH
 import com.tencent.bkrepo.npm.constants.NPM_PACKAGE_TGZ_FILE
+import com.tencent.bkrepo.npm.constants.REQUEST_URI
 import com.tencent.bkrepo.npm.constants.SEARCH_REQUEST
 import com.tencent.bkrepo.npm.constants.SIZE
 import com.tencent.bkrepo.npm.exception.NpmArtifactExistException
@@ -275,6 +276,7 @@ class NpmClientServiceImpl(
             val context = ArtifactQueryContext()
             val packageFullPath = NpmUtils.getPackageMetadataPath(name)
             context.putAttribute(NPM_FILE_FULL_PATH, packageFullPath)
+            context.putAttribute(REQUEST_URI, name)
             val inputStream =
                 ArtifactContextHolder.getRepository().query(context) as? InputStream
                     ?: throw NpmArtifactNotFoundException("document not found")
@@ -304,6 +306,7 @@ class NpmClientServiceImpl(
             val context = ArtifactQueryContext()
             val packageFullPath = NpmUtils.getVersionPackageMetadataPath(name, version)
             context.putAttribute(NPM_FILE_FULL_PATH, packageFullPath)
+            context.putAttribute(REQUEST_URI, "$name/$version")
             val inputStream =
                 ArtifactContextHolder.getRepository().query(context) as? InputStream
                     ?: throw NpmArtifactNotFoundException("document not found")

src/backend/oci/biz-oci/src/main/kotlin/com/tencent/bkrepo/oci/artifact/repository/OciRegistryRemoteRepository.kt

@@ -453,7 +453,7 @@ class OciRegistryRemoteRepository(
     /**
      * 尝试获取缓存的远程构件节点
      */
-    override fun findCacheNodeDetail(context: ArtifactDownloadContext): NodeDetail? {
+    override fun findCacheNodeDetail(context: ArtifactContext): NodeDetail? {
         with(context) {
             val fullPath = ociOperationService.getNodeFullPath(context.artifactInfo as OciArtifactInfo) ?: return null
             return nodeClient.getNodeDetail(projectId, repoName, fullPath).data
@@ -463,7 +463,7 @@ class OciRegistryRemoteRepository(
     /**
      * 加载要返回的资源: oci协议需要返回特定的请求头和资源类型
      */
-    override fun loadArtifactResource(cacheNode: NodeDetail, context: ArtifactDownloadContext): ArtifactResource? {
+    override fun loadArtifactResource(cacheNode: NodeDetail, context: ArtifactContext): ArtifactResource? {
         return storageService.load(cacheNode.sha256!!, Range.full(cacheNode.size), context.storageCredentials)?.run {
             if (logger.isDebugEnabled) {
                 logger.debug("Cached remote artifact[${context.artifactInfo}] is hit.")
@@ -484,7 +484,7 @@ class OciRegistryRemoteRepository(
 
     private fun buildResponse(
         cacheNode: NodeDetail?,
-        context: ArtifactDownloadContext,
+        context: ArtifactContext,
         artifactResource: ArtifactResource,
         sha256: String? = null,
         size: Long? = null