diff --git a/src/backend/buildSrc/src/main/kotlin/Versions.kt b/src/backend/buildSrc/src/main/kotlin/Versions.kt index e581f1a229..459a0d6829 100644 --- a/src/backend/buildSrc/src/main/kotlin/Versions.kt +++ b/src/backend/buildSrc/src/main/kotlin/Versions.kt @@ -76,5 +76,4 @@ object Versions { const val Galimatias = "0.2.1" const val CommonsNet = "3.9.0" const val JuniversalCharDet = "1.0.3" - const val HttpClient = "4.5.13" } diff --git a/src/backend/preview/biz-preview/build.gradle b/src/backend/preview/biz-preview/build.gradle index c727271606..d388aa14b1 100644 --- a/src/backend/preview/biz-preview/build.gradle +++ b/src/backend/preview/biz-preview/build.gradle @@ -33,7 +33,6 @@ dependencies { //api(project(":common:common-generic")) api(project(":common:common-artifact:artifact-service")) api("org.dom4j:dom4j:${Versions.Dom4j}") - api("org.apache.httpcomponents:httpclient:${Versions.HttpClient}") // office转换 api("org.jodconverter:jodconverter-local:${Versions.JodConverter}") // url规范化 diff --git a/src/backend/preview/biz-preview/src/main/kotlin/com/tencent/bkrepo/preview/config/configuration/PreviewConfig.kt b/src/backend/preview/biz-preview/src/main/kotlin/com/tencent/bkrepo/preview/config/configuration/PreviewConfig.kt index fe06d5db7d..79faf489c0 100644 --- a/src/backend/preview/biz-preview/src/main/kotlin/com/tencent/bkrepo/preview/config/configuration/PreviewConfig.kt +++ b/src/backend/preview/biz-preview/src/main/kotlin/com/tencent/bkrepo/preview/config/configuration/PreviewConfig.kt @@ -49,6 +49,12 @@ class PreviewConfig { @Value("\${preview.repoName:convert}") val repoName = "convert" + /** + * repo是否公开 + */ + @Value("\${preview.repoPublic:true}") + val repoPublic = true + /** * 临时文件保存路径 */ diff --git a/src/backend/preview/biz-preview/src/main/kotlin/com/tencent/bkrepo/preview/config/startup/PreviewStartupRunner.kt b/src/backend/preview/biz-preview/src/main/kotlin/com/tencent/bkrepo/preview/config/startup/PreviewStartupRunner.kt index e6e71b1d9e..27c5e65c05 100644 --- a/src/backend/preview/biz-preview/src/main/kotlin/com/tencent/bkrepo/preview/config/startup/PreviewStartupRunner.kt +++ b/src/backend/preview/biz-preview/src/main/kotlin/com/tencent/bkrepo/preview/config/startup/PreviewStartupRunner.kt @@ -37,8 +37,10 @@ import com.tencent.bkrepo.common.artifact.pojo.configuration.local.LocalConfigur import com.tencent.bkrepo.common.metadata.service.project.ProjectService import com.tencent.bkrepo.common.metadata.service.repo.RepositoryService import com.tencent.bkrepo.preview.config.configuration.PreviewConfig +import com.tencent.bkrepo.repository.constant.SYSTEM_USER import com.tencent.bkrepo.repository.pojo.project.ProjectCreateRequest import com.tencent.bkrepo.repository.pojo.repo.RepoCreateRequest +import com.tencent.bkrepo.repository.pojo.repo.RepoUpdateRequest import org.slf4j.Logger import org.slf4j.LoggerFactory import org.springframework.boot.ApplicationArguments @@ -74,27 +76,36 @@ class PreviewStartupRunner( val projectId = config.projectId val repoName = config.repoName var exist = repositoryService.checkExist(projectId, repoName, RepositoryType.GENERIC.name) - if (!exist) { - val repoConfig = LocalConfiguration() - val cleanupStrategy = mutableMapOf( - "enable" to true, - "cleanupType" to "retentionDays", - "cleanupValue" to config.artifactKeepDays - ) - repoConfig.settings["cleanupStrategy"] = cleanupStrategy + val repoConfig = LocalConfiguration() + val cleanupStrategy = mutableMapOf( + "enable" to true, + "cleanupType" to "retentionDays", + "cleanupValue" to config.artifactKeepDays + ) + repoConfig.settings["cleanupStrategy"] = cleanupStrategy + if (!exist) { val req = RepoCreateRequest(projectId = config.projectId, name = config.repoName, type = RepositoryType.GENERIC, category = RepositoryCategory.LOCAL, - public = true, + public = config.repoPublic, quota = config.repoQuota * 1024 * 1024, configuration =repoConfig ) var createdRepo = repositoryService.createRepo(req) logger.debug("Create project success，projectId:${createdRepo.name}") } else { - logger.debug("project ${config.projectId} and repository ${config.repoName} exist skip.") + logger.debug("project ${config.projectId} and repository ${config.repoName} exist. to update") + val updateRepo = RepoUpdateRequest( + projectId = config.projectId, + name = config.repoName, + public = config.repoPublic, + quota = config.repoQuota * 1024 * 1024, + configuration =repoConfig, + operator = SYSTEM_USER + ) + repositoryService.updateRepo(updateRepo) } } diff --git a/src/backend/preview/biz-preview/src/main/kotlin/com/tencent/bkrepo/preview/dao/FilePreviewCacheDao.kt b/src/backend/preview/biz-preview/src/main/kotlin/com/tencent/bkrepo/preview/dao/FilePreviewCacheDao.kt index 90a9df845a..ccf148c26e 100644 --- a/src/backend/preview/biz-preview/src/main/kotlin/com/tencent/bkrepo/preview/dao/FilePreviewCacheDao.kt +++ b/src/backend/preview/biz-preview/src/main/kotlin/com/tencent/bkrepo/preview/dao/FilePreviewCacheDao.kt @@ -34,6 +34,7 @@ package com.tencent.bkrepo.preview.dao import com.mongodb.client.result.DeleteResult import com.tencent.bkrepo.common.mongo.dao.simple.SimpleMongoDao import com.tencent.bkrepo.preview.model.TPreviewFileCache +import org.springframework.data.mongodb.core.query.Criteria import org.springframework.data.mongodb.core.query.Query import org.springframework.data.mongodb.core.query.isEqualTo import org.springframework.stereotype.Repository @@ -41,15 +42,22 @@ import org.springframework.stereotype.Repository @Repository class FilePreviewCacheDao : SimpleMongoDao() { /** - * 按md5查找 + * 查找缓存 */ - fun findByMd5(md5: String): TPreviewFileCache? { - return this.findOne(Query(TPreviewFileCache::md5.isEqualTo(md5))) + fun getCache(md5: String, projectId: String, repoName: String): TPreviewFileCache? { + return this.findOne(Query(buildCriteria(md5, projectId, repoName))) } /** - * 按md5删除 + * 删除缓存 */ - fun removeByMd5(md5: String): DeleteResult { - return this.remove(Query(TPreviewFileCache::md5.isEqualTo(md5))) + fun removeCache(md5: String, projectId: String, repoName: String): DeleteResult { + return this.remove(Query(buildCriteria(md5, projectId, repoName))) + } + + private fun buildCriteria(md5: String, projectId: String, repoName: String): Criteria { + return Criteria + .where(TPreviewFileCache::md5.name).isEqualTo(md5) + .and(TPreviewFileCache::projectId.name).isEqualTo(projectId) + .and(TPreviewFileCache::repoName.name).isEqualTo(repoName) } } \ No newline at end of file diff --git a/src/backend/preview/biz-preview/src/main/kotlin/com/tencent/bkrepo/preview/service/CommonResourceService.kt b/src/backend/preview/biz-preview/src/main/kotlin/com/tencent/bkrepo/preview/service/CommonResourceService.kt index 1b98bc98ee..e1aee46e99 100644 --- a/src/backend/preview/biz-preview/src/main/kotlin/com/tencent/bkrepo/preview/service/CommonResourceService.kt +++ b/src/backend/preview/biz-preview/src/main/kotlin/com/tencent/bkrepo/preview/service/CommonResourceService.kt @@ -1,9 +1,9 @@ package com.tencent.bkrepo.preview.service +import com.tencent.bkrepo.common.api.util.readJsonString import com.tencent.bkrepo.preview.config.configuration.PreviewConfig import com.tencent.bkrepo.preview.pojo.PreviewOptions import com.tencent.bkrepo.preview.pojo.Watermark -import com.tencent.bkrepo.preview.utils.JsonMapper import org.slf4j.Logger import org.slf4j.LoggerFactory import org.springframework.stereotype.Component @@ -19,9 +19,8 @@ class CommonResourceService(private val config: PreviewConfig) { * 水印 */ fun getWatermark(decodedParams: String?): Watermark { - val jsonMapper = JsonMapper() val watermark = if (!decodedParams.isNullOrEmpty()) { - jsonMapper.fromJson(decodedParams, Watermark::class.java)?: Watermark() + decodedParams!!.readJsonString() } else { Watermark() } diff --git a/src/backend/preview/biz-preview/src/main/kotlin/com/tencent/bkrepo/preview/service/FileHandlerService.kt b/src/backend/preview/biz-preview/src/main/kotlin/com/tencent/bkrepo/preview/service/FileHandlerService.kt index 350729aaa9..4df6ab6af6 100644 --- a/src/backend/preview/biz-preview/src/main/kotlin/com/tencent/bkrepo/preview/service/FileHandlerService.kt +++ b/src/backend/preview/biz-preview/src/main/kotlin/com/tencent/bkrepo/preview/service/FileHandlerService.kt @@ -1,5 +1,6 @@ package com.tencent.bkrepo.preview.service +import com.tencent.bkrepo.common.api.util.readJsonString import com.tencent.bkrepo.common.artifact.api.ArtifactInfo import com.tencent.bkrepo.preview.config.configuration.PreviewConfig import com.tencent.bkrepo.preview.constant.PreviewMessageCode @@ -8,7 +9,6 @@ import com.tencent.bkrepo.preview.pojo.FileAttribute import com.tencent.bkrepo.preview.pojo.FileType import com.tencent.bkrepo.preview.pojo.PreviewInfo import com.tencent.bkrepo.preview.utils.FileUtils -import com.tencent.bkrepo.preview.utils.JsonMapper import com.tencent.bkrepo.preview.utils.UrlEncoderUtils import com.tencent.bkrepo.preview.utils.WebUtils import org.slf4j.LoggerFactory @@ -73,7 +73,6 @@ class FileHandlerService( */ fun getFileTemplate(fileAttribute: FileAttribute): String = with(fileAttribute) { when { - isHtmlView && suffix?.lowercase() == "csv" -> FilePreview.CSV_FILE_PREVIEW_PAGE isHtmlView -> FilePreview.EXEL_FILE_PREVIEW_PAGE type?.name == FileType.OFFICE.name -> when (suffix!!.lowercase()) { "xlsx" -> FilePreview.XLSX_FILE_PREVIEW_PAGE @@ -99,8 +98,7 @@ class FileHandlerService( * @return 文件属性 */ fun getFileAttribute(params: String): FileAttribute { - val jsonMapper = JsonMapper() - val attribute = jsonMapper.fromJson(params, FileAttribute::class.java)?: FileAttribute() + val attribute = params.readJsonString() checkRequest(attribute) adjustProperties(attribute) return attribute @@ -113,8 +111,7 @@ class FileHandlerService( * @return 文件属性 */ fun getFileAttribute(artifactInfo: ArtifactInfo, params: String?): FileAttribute { - val jsonMapper = JsonMapper() - val attribute = jsonMapper.fromJson(params, FileAttribute::class.java)?: FileAttribute() + val attribute = params?.readJsonString() ?: FileAttribute() attribute.projectId = artifactInfo.projectId attribute.repoName = artifactInfo.repoName attribute.artifactUri = artifactInfo.getArtifactFullPath() @@ -265,7 +262,7 @@ class FileHandlerService( // 判断是否为HTML视图 private fun isHtmlView(suffix: String): Boolean { - val htmlSuffixes = setOf("xls", "csv", "xlsm", "xlt", "xltm", "et", "ett", "xlam") + val htmlSuffixes = setOf("xls", "xlsm", "xlt", "xltm", "et", "ett", "xlam") return htmlSuffixes.contains(suffix.lowercase()) } } diff --git a/src/backend/preview/biz-preview/src/main/kotlin/com/tencent/bkrepo/preview/service/OfficePluginManager.kt b/src/backend/preview/biz-preview/src/main/kotlin/com/tencent/bkrepo/preview/service/OfficePluginManager.kt index 3e5a4c4bed..9d1fa80a0f 100644 --- a/src/backend/preview/biz-preview/src/main/kotlin/com/tencent/bkrepo/preview/service/OfficePluginManager.kt +++ b/src/backend/preview/biz-preview/src/main/kotlin/com/tencent/bkrepo/preview/service/OfficePluginManager.kt @@ -31,7 +31,6 @@ package com.tencent.bkrepo.preview.service -import javax.annotation.PostConstruct import javax.annotation.PreDestroy import org.apache.commons.lang3.StringUtils import org.jodconverter.core.office.InstalledOfficeManagerHolder @@ -52,7 +51,7 @@ import java.io.IOException */ @Component class OfficePluginManager { - private lateinit var officeManager: LocalOfficeManager + private var officeManager: LocalOfficeManager ? = null @Value("\${preview.office.plugin.server.ports:2001,2002}") private val serverPorts = "2001,2002" @@ -69,11 +68,16 @@ class OfficePluginManager { @Value("\${preview.office.home:/opt/libreoffice7.6}") private val officeHome: String = "/opt/libreoffice7.6" + fun startOfficeManagerIfNeeded() { + if (officeManager == null) { + startOfficeManager() + } + } + /** * 启动Office组件进程 */ - @PostConstruct - fun startOfficeManager() { + private fun startOfficeManager() { val officeHome: File = getOfficeHome(officeHome) logger.info("Office component path：${officeHome.path}") val killOffice = killProcess() @@ -92,7 +96,7 @@ class OfficePluginManager { .maxTasksPerProcess(maxTasksPerProcess) .taskExecutionTimeout(taskexecutiontimeout) .build() - officeManager.start() + officeManager?.start() InstalledOfficeManagerHolder.setInstance(officeManager) } catch (e: Exception) { logger.error("The office component fails to start, check whether the office component is available",e) @@ -165,7 +169,7 @@ class OfficePluginManager { @PreDestroy fun destroyOfficeManager() { - if (null != officeManager && officeManager.isRunning()) { + if (null != officeManager && officeManager!!.isRunning()) { logger.info("Shutting down office process") OfficeUtils.stopQuietly(officeManager) } diff --git a/src/backend/preview/biz-preview/src/main/kotlin/com/tencent/bkrepo/preview/service/OfficeToPdfService.kt b/src/backend/preview/biz-preview/src/main/kotlin/com/tencent/bkrepo/preview/service/OfficeToPdfService.kt index a8e5cf50a2..12e65ce9f7 100644 --- a/src/backend/preview/biz-preview/src/main/kotlin/com/tencent/bkrepo/preview/service/OfficeToPdfService.kt +++ b/src/backend/preview/biz-preview/src/main/kotlin/com/tencent/bkrepo/preview/service/OfficeToPdfService.kt @@ -46,7 +46,10 @@ import java.io.File * office转pdf服务 */ @Component -class OfficeToPdfService(private val config: PreviewConfig) { +class OfficeToPdfService( + private val config: PreviewConfig, + private val officePluginManager: OfficePluginManager +) { companion object { private val logger: Logger = LoggerFactory.getLogger(OfficeToPdfService::class.java) @@ -54,6 +57,7 @@ class OfficeToPdfService(private val config: PreviewConfig) { @Throws(OfficeException::class) fun openOfficeToPDF(inputFilePath: String, outputFilePath: String, fileAttribute: FileAttribute) { + officePluginManager.startOfficeManagerIfNeeded() office2pdf(inputFilePath, outputFilePath, fileAttribute) } diff --git a/src/backend/preview/biz-preview/src/main/kotlin/com/tencent/bkrepo/preview/service/cache/PreviewFileCacheService.kt b/src/backend/preview/biz-preview/src/main/kotlin/com/tencent/bkrepo/preview/service/cache/PreviewFileCacheService.kt index 5b06d929a6..3f34933d11 100644 --- a/src/backend/preview/biz-preview/src/main/kotlin/com/tencent/bkrepo/preview/service/cache/PreviewFileCacheService.kt +++ b/src/backend/preview/biz-preview/src/main/kotlin/com/tencent/bkrepo/preview/service/cache/PreviewFileCacheService.kt @@ -39,8 +39,8 @@ interface PreviewFileCacheService { fun createCache(requestFile: PreviewFileCacheCreateRequest): PreviewFileCacheInfo // 删除缓存 - fun removeCacheByMd5(md5: String) + fun removeCache(md5: String, projectId: String, repoName: String) - // 根据md5查询缓存 - fun getCacheByMd5(md5: String): PreviewFileCacheInfo? + // 查询缓存 + fun getCache(md5: String, projectId: String, repoName: String): PreviewFileCacheInfo? } \ No newline at end of file diff --git a/src/backend/preview/biz-preview/src/main/kotlin/com/tencent/bkrepo/preview/service/cache/impl/PreviewFileCacheServiceImpl.kt b/src/backend/preview/biz-preview/src/main/kotlin/com/tencent/bkrepo/preview/service/cache/impl/PreviewFileCacheServiceImpl.kt index c5d0bf165e..0045162ce8 100644 --- a/src/backend/preview/biz-preview/src/main/kotlin/com/tencent/bkrepo/preview/service/cache/impl/PreviewFileCacheServiceImpl.kt +++ b/src/backend/preview/biz-preview/src/main/kotlin/com/tencent/bkrepo/preview/service/cache/impl/PreviewFileCacheServiceImpl.kt @@ -57,12 +57,12 @@ class PreviewFileCacheServiceImpl( return convert(filePreviewCache) } - override fun removeCacheByMd5(md5: String) { - filePreviewCacheDao.removeByMd5(md5) + override fun removeCache(md5: String, projectId: String, repoName: String) { + filePreviewCacheDao.removeCache(md5, projectId, repoName) } - override fun getCacheByMd5(md5: String): PreviewFileCacheInfo? { - return filePreviewCacheDao.findByMd5(md5)?.let { convert(it) } + override fun getCache(md5: String, projectId: String, repoName: String): PreviewFileCacheInfo? { + return filePreviewCacheDao.getCache(md5, projectId, repoName)?.let { convert(it) } } private fun convert(previewFileCache: TPreviewFileCache): PreviewFileCacheInfo { diff --git a/src/backend/preview/biz-preview/src/main/kotlin/com/tencent/bkrepo/preview/service/impl/AbstractFilePreview.kt b/src/backend/preview/biz-preview/src/main/kotlin/com/tencent/bkrepo/preview/service/impl/AbstractFilePreview.kt index 4eb7fe86e3..b879c71dd6 100644 --- a/src/backend/preview/biz-preview/src/main/kotlin/com/tencent/bkrepo/preview/service/impl/AbstractFilePreview.kt +++ b/src/backend/preview/biz-preview/src/main/kotlin/com/tencent/bkrepo/preview/service/impl/AbstractFilePreview.kt @@ -73,8 +73,9 @@ abstract class AbstractFilePreview( val downloadResult = downloadFile(fileAttribute) ?: throw PreviewNotFoundException(PreviewMessageCode.PREVIEW_FILE_NOT_FOUND, fileAttribute.fileName!!) - // 从缓存获取最终文件并且节点是否存在 - var previewFileCacheInfo = if (config.cacheEnabled) getCacheAndCheckExist(fileAttribute.md5!!) else null + // 从缓存获取最终文件并且判断节点是否存在 + var previewFileCacheInfo = if (config.cacheEnabled) getCacheAndCheckExist(fileAttribute) else null + if (previewFileCacheInfo == null) { // 文件校验，比如是否超过最大预览限制 checkFileConstraints(fileAttribute) @@ -147,8 +148,11 @@ abstract class AbstractFilePreview( /** * 获取预览文件缓存 */ - private fun getCacheAndCheckExist(md5: String): PreviewFileCacheInfo? { - val filePreviewCacheInfo = previewFileCacheService.getCacheByMd5(md5) ?: return null + private fun getCacheAndCheckExist(fileAttribute: FileAttribute): PreviewFileCacheInfo? { + val projectId = if (fileAttribute.storageType == 0) fileAttribute.projectId else config.projectId + val repoName = if (fileAttribute.storageType == 0) fileAttribute.repoName else config.repoName + val md5 = fileAttribute.md5 + val filePreviewCacheInfo = previewFileCacheService.getCache(md5!!, projectId!!, repoName!!) ?: return null // 检查节点是否存在 return if (nodeService.checkExist( ArtifactInfo( @@ -161,7 +165,7 @@ abstract class AbstractFilePreview( filePreviewCacheInfo } else { // 节点不存在，移除缓存 - previewFileCacheService.removeCacheByMd5(md5) + previewFileCacheService.removeCache(md5, projectId, repoName) logger.warn("node does not exist, delete the cache information, key：$md5") null } diff --git a/src/backend/preview/biz-preview/src/main/kotlin/com/tencent/bkrepo/preview/utils/DownloadUtils.kt b/src/backend/preview/biz-preview/src/main/kotlin/com/tencent/bkrepo/preview/utils/DownloadUtils.kt index 7349faa03e..000e8ff959 100644 --- a/src/backend/preview/biz-preview/src/main/kotlin/com/tencent/bkrepo/preview/utils/DownloadUtils.kt +++ b/src/backend/preview/biz-preview/src/main/kotlin/com/tencent/bkrepo/preview/utils/DownloadUtils.kt @@ -31,23 +31,24 @@ package com.tencent.bkrepo.preview.utils +import com.tencent.bkrepo.common.api.constant.HttpHeaders +import com.tencent.bkrepo.common.api.constant.MediaTypes import com.tencent.bkrepo.common.api.exception.SystemErrorException import com.tencent.bkrepo.common.api.message.CommonMessageCode +import com.tencent.bkrepo.common.service.util.okhttp.HttpClientBuilderFactory import com.tencent.bkrepo.preview.config.configuration.PreviewConfig import com.tencent.bkrepo.preview.pojo.DownloadResult import com.tencent.bkrepo.preview.pojo.FileAttribute -import org.apache.http.impl.client.DefaultRedirectStrategy -import org.apache.http.impl.client.HttpClientBuilder -import org.springframework.http.HttpMethod -import org.springframework.http.MediaType -import org.springframework.http.client.HttpComponentsClientHttpRequestFactory -import org.springframework.web.client.RequestCallback -import org.springframework.web.client.RestTemplate +import okhttp3.OkHttpClient +import okhttp3.Request +import okhttp3.ResponseBody import java.io.File import java.io.FileNotFoundException import java.io.IOException +import java.net.SocketTimeoutException import java.net.URL import java.util.UUID +import java.util.concurrent.TimeUnit /** * 文件下载工具 @@ -57,8 +58,6 @@ object DownloadUtils { private const val URL_PARAM_FTP_USERNAME = "ftp.username" private const val URL_PARAM_FTP_PASSWORD = "ftp.password" private const val URL_PARAM_FTP_CONTROL_ENCODING = "ftp.control.encoding" - private val restTemplate = RestTemplate() - private val factory = HttpComponentsClientHttpRequestFactory() /** * 下载文件 @@ -104,25 +103,13 @@ object DownloadUtils { } private fun downloadHttpFile(url: URL, realPath: String, result: DownloadResult) { - try { - val realFile = File(realPath) - factory.setReadTimeout(72000) - factory.setConnectTimeout(10000) - factory.setConnectionRequestTimeout(2000) - factory.httpClient = HttpClientBuilder.create() - .setRedirectStrategy(DefaultRedirectStrategy()) - .build() - restTemplate.requestFactory = factory - val requestCallback = RequestCallback { request -> - request.headers.setAccept(listOf(MediaType.APPLICATION_OCTET_STREAM, MediaType.ALL)) - } + val client = createHttpClient() + val request = createRequest(url) - restTemplate.execute(url.toURI(), HttpMethod.GET, requestCallback) { fileResponse -> - org.apache.commons.io.FileUtils.copyToFile(fileResponse.body, realFile) - null - } + try { + retryDownload(request, client, realPath, result) } catch (e: Exception) { - logger.error("The download failed: $e") + logger.error("Download failed: $e") result.apply { code = DownloadResult.CODE_FAIL msg = "The download failed: $e" @@ -130,6 +117,76 @@ object DownloadUtils { } } + private fun createHttpClient(): OkHttpClient { + return HttpClientBuilderFactory + .create() + .readTimeout(72000, TimeUnit.MILLISECONDS) + .connectTimeout(10000, TimeUnit.MILLISECONDS) + .retryOnConnectionFailure(true) + .build() + } + + private fun createRequest(url: URL): Request { + return Request.Builder() + .url(url) + .addHeader(HttpHeaders.ACCEPT, MediaTypes.APPLICATION_OCTET_STREAM) + .build() + } + + private fun retryDownload(request: Request, client: OkHttpClient, realPath: String, result: DownloadResult) { + val maxRetries = 3 + var attempt = 0 + + while (attempt < maxRetries) { + try { + val response = client.newCall(request).execute() + if (response.isSuccessful) { + saveFile(response.body, realPath) + result.apply { + code = DownloadResult.CODE_SUCCESS + msg = "Download succeeded." + } + return + } else { + throw IOException("Failed to download file: ${response.code}") + } + } catch (e: Exception) { + attempt++ + if (attempt >= maxRetries) { + logger.error("Download failed after $attempt attempts: $e") + result.apply { + code = DownloadResult.CODE_FAIL + msg = "The download failed after $attempt attempts: $e" + } + return + } + if (shouldRetry(e)) { + logger.warn("Retrying download due to error: $e (Attempt: $attempt)") + Thread.sleep(2000) //等待2秒后重试 + } else { + throw e + } + } + } + } + + private fun saveFile(body: ResponseBody?, realPath: String) { + val file = File(realPath) + if (!file.parentFile.exists() && !file.parentFile.mkdirs()) { + logger.error("Failed to create directory [$realPath], please check the directory permissions!") + throw SystemErrorException(CommonMessageCode.SYSTEM_ERROR, "Failed to create directory") + } + body?.byteStream()?.use { inputStream -> + file.outputStream().use { outputStream -> + inputStream.copyTo(outputStream) + } + } + } + + private fun shouldRetry(e: Exception): Boolean { + return e is IOException || e is SocketTimeoutException + } + private fun downloadFtpFile(fileAttribute: FileAttribute, realPath: String, config: PreviewConfig, @@ -196,10 +253,9 @@ object DownloadUtils { private fun processUrl(fileAttribute: FileAttribute): String? { return try { - SslUtils.ignoreSsl() fileAttribute.url?.replace("+", "%20")?.replace(" ", "%20") } catch (e: Exception) { - logger.error("Ignore SSL certificate exceptions", e) + logger.error("processUrl exceptions", e) null } } diff --git a/src/backend/preview/biz-preview/src/main/kotlin/com/tencent/bkrepo/preview/utils/JsonMapper.kt b/src/backend/preview/biz-preview/src/main/kotlin/com/tencent/bkrepo/preview/utils/JsonMapper.kt deleted file mode 100644 index 5b535517d5..0000000000 --- a/src/backend/preview/biz-preview/src/main/kotlin/com/tencent/bkrepo/preview/utils/JsonMapper.kt +++ /dev/null @@ -1,132 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. - * - * Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. - * - * A copy of the MIT License is included in this file. - * - * - * Terms of the MIT License: - * --------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -package com.tencent.bkrepo.preview.utils - -import com.fasterxml.jackson.annotation.JsonInclude.Include -import com.fasterxml.jackson.core.type.TypeReference -import com.fasterxml.jackson.databind.DeserializationFeature -import com.fasterxml.jackson.databind.ObjectMapper -import com.fasterxml.jackson.databind.SerializationFeature -import org.slf4j.LoggerFactory -import java.io.IOException -import java.text.SimpleDateFormat - -/** - * 简单封装Jackson，定制不同的输出 - */ -class JsonMapper @JvmOverloads constructor(include: Include? = null) { - - private val mapper: ObjectMapper = ObjectMapper() - - init { - // 设置输出时包含属性的风格 - include?.let { mapper.setSerializationInclusion(it) } - // 设置输入时忽略在JSON字符串中存在但Java对象实际没有的属性 - mapper.disable(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES) - mapper.disable(SerializationFeature.FAIL_ON_EMPTY_BEANS) - mapper.dateFormat = SimpleDateFormat("yyyy-MM-dd HH:mm:ss") - } - - companion object { - private val logger = LoggerFactory.getLogger(JsonMapper::class.java) - - // 非空属性的序列化 - fun nonEmptyMapper(): JsonMapper = JsonMapper(Include.NON_EMPTY) - - // 非默认值属性的序列化 - fun nonDefaultMapper(): JsonMapper = JsonMapper(Include.NON_DEFAULT) - - // 全部属性序列化 - fun getAllOutputMapper(): JsonMapper = JsonMapper(Include.ALWAYS) - } - - /** - * 将对象转换为 JSON 字符串 - */ - fun toJson(obj: Any?): String? { - return try { - mapper.writeValueAsString(obj) - } catch (e: IOException) { - logger.warn("object to json string error.", e); - null - } - } - - /** - * 反序列化 POJO 或简单集合，如 List。 - * 如果 JSON 字符串为 Null 或 "null" 字符串，返回 Null。 - * 如果 JSON 字符串为 "[]"，返回空集合。 - * 如需反序列化复杂集合，如 List，请使用 fromJson(String, TypeReference)。 - */ - fun fromJson(jsonString: String?, clazz: Class): T? { - if (jsonString.isNullOrEmpty()) return null - - return try { - mapper.readValue(jsonString, clazz) - } catch (e: IOException) { - logger.warn("Parse json string error.", e) - null - } - } - - /** - * 反序列化复杂的泛型对象 - * 如果 JSON 字符串为 Null 或 "null" 字符串，返回 Null。 - * 如果 JSON 字符串为 "[]"，返回空集合。 - */ - fun fromJson(jsonString: String?, typeReference: TypeReference): T? { - if (jsonString.isNullOrEmpty()) return null - - return try { - mapper.readValue(jsonString, typeReference) - } catch (e: IOException) { - logger.warn("Parse json string error.", e) - null - } - } - - /** - * 当 JSON 里只含有 Bean 的部分属性时，更新一个已存在 Bean，只覆盖该部分的属性。 - */ - fun update(jsonString: String?, obj: Any) { - try { - mapper.readerForUpdating(obj).readValue(jsonString) - } catch (e: IOException) { - logger.warn("Update json string to object error.", e) - } - } - - /** - * 获取底层的 ObjectMapper，用于进一步的设置或使用其他序列化 API。 - */ - fun getMapper(): ObjectMapper = mapper -} \ No newline at end of file diff --git a/src/backend/preview/biz-preview/src/main/kotlin/com/tencent/bkrepo/preview/utils/SslUtils.kt b/src/backend/preview/biz-preview/src/main/kotlin/com/tencent/bkrepo/preview/utils/SslUtils.kt deleted file mode 100644 index 34b45abc07..0000000000 --- a/src/backend/preview/biz-preview/src/main/kotlin/com/tencent/bkrepo/preview/utils/SslUtils.kt +++ /dev/null @@ -1,80 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. - * - * Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. - * - * A copy of the MIT License is included in this file. - * - * - * Terms of the MIT License: - * --------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -package com.tencent.bkrepo.preview.utils - -import java.security.cert.X509Certificate -import javax.net.ssl.HostnameVerifier -import javax.net.ssl.HttpsURLConnection -import javax.net.ssl.TrustManager -import javax.net.ssl.X509TrustManager -import javax.net.ssl.SSLContext - -/** - * SSL工具 - */ -object SslUtils { - - /** - * 忽略所有HTTPS请求的SSL证书 - * 必须在openConnection之前调用 - */ - @Throws(Exception::class) - fun ignoreSsl() { - val hv = HostnameVerifier { _, _ -> true } - trustAllHttpsCertificates() - HttpsURLConnection.setDefaultHostnameVerifier(hv) - } - - /** - * 信任所有HTTPS证书 - */ - @Throws(Exception::class) - private fun trustAllHttpsCertificates() { - val trustAllCerts = arrayOf(miTM()) - val sc = SSLContext.getInstance("SSL") - sc.init(null, trustAllCerts, null) - HttpsURLConnection.setDefaultSSLSocketFactory(sc.socketFactory) - } - - // 自定义 TrustManager 实现 - private class miTM : X509TrustManager { - override fun getAcceptedIssuers(): Array? = null - - override fun checkServerTrusted(certs: Array, authType: String) { - // No-op: Accepting all certificates - } - - override fun checkClientTrusted(certs: Array, authType: String) { - // No-op: Accepting all certificates - } - } -} \ No newline at end of file diff --git a/support-files/kubernetes/charts/bkrepo/values.yaml b/support-files/kubernetes/charts/bkrepo/values.yaml index b3937f805b..533eb1530f 100644 --- a/support-files/kubernetes/charts/bkrepo/values.yaml +++ b/support-files/kubernetes/charts/bkrepo/values.yaml @@ -547,7 +547,7 @@ s3: ## preview服务配置 preview: - enabled: true + enabled: false config: {} ## Kubernetes 通用配置 image: