From 77075281b537aa347bd36e7325a47643277f1309 Mon Sep 17 00:00:00 2001 From: George Andrinopoulos Date: Thu, 24 Aug 2023 15:55:10 +0300 Subject: [PATCH] CID-1866 Separate ORT produced files retention from repository files --- README.md | 2 +- .../sbomBooster/domain/VsmDiscoveryItem.kt | 2 +- .../vsm/sbomBooster/service/OrtService.kt | 22 +++++++++++++------ .../vsm/sbomBooster/service/ProcessService.kt | 22 ++++++++++++------- .../service/VsmDiscoveryService.kt | 4 ++-- 5 files changed, 33 insertions(+), 19 deletions(-) diff --git a/README.md b/README.md index 60148a3..861312a 100644 --- a/README.md +++ b/README.md @@ -93,7 +93,7 @@ The second `-v` param is the path to temporary folder that the `vsm-sbom-booster `ANALYSIS_TIMEOUT`(optional): The timeout, in minutes, that is used to force kill container workers working on analyzing the repository. There are cases that we need to force kill containers with slow progress to free up resources. Default: 30 -`DEV_MODE`(optional): This is a flag to enable/disable the dev mode. When enabled, all logs from ORT containers will be preserved in the temp folder location and the project folders will be retained. Additionally, the logging level for the ORT containers will be set to DEBUG. The processing of each repository consists of three separate phases (download, analyze and generate_sbom) that are facilitated by the ORT software. The produced logs are saved in the temp folder using the `_.txt` naming pattern.This is useful for debugging purposes. Default: false +`DEV_MODE`(optional): This is a flag to enable/disable the dev mode. When enabled, all logs from ORT containers will be preserved in the temp folder location and the ORT project folders will be retained. Additionally, the logging level for the ORT containers will be set to DEBUG. The processing of each repository consists of three separate phases (download, analyze and generate_sbom) that are facilitated by the ORT software. The produced logs are saved in the temp folder using the `__log.txt` naming pattern.This is useful for debugging purposes. Default: false #### LeanIX configs diff --git a/src/main/kotlin/net/leanix/vsm/sbomBooster/domain/VsmDiscoveryItem.kt b/src/main/kotlin/net/leanix/vsm/sbomBooster/domain/VsmDiscoveryItem.kt index ab075fd..a086698 100644 --- a/src/main/kotlin/net/leanix/vsm/sbomBooster/domain/VsmDiscoveryItem.kt +++ b/src/main/kotlin/net/leanix/vsm/sbomBooster/domain/VsmDiscoveryItem.kt @@ -2,7 +2,7 @@ package net.leanix.vsm.sbomBooster.domain data class VsmDiscoveryItem( val projectUrl: String, - val downloadedFolder: String, + val ortFolder: String, val sourceType: String, val sourceInstance: String, val name: String, diff --git a/src/main/kotlin/net/leanix/vsm/sbomBooster/service/OrtService.kt b/src/main/kotlin/net/leanix/vsm/sbomBooster/service/OrtService.kt index d337477..3d466d2 100644 --- a/src/main/kotlin/net/leanix/vsm/sbomBooster/service/OrtService.kt +++ b/src/main/kotlin/net/leanix/vsm/sbomBooster/service/OrtService.kt @@ -54,10 +54,16 @@ class OrtService( return downloadFolder } - fun analyzeProject(projectUrl: String, downloadFolder: String) { + fun analyzeProject(projectUrl: String, downloadFolder: String): String { + + val ortFolder = "${projectUrl.substringAfterLast("/")}_ORT_produced_files" + val analyzeProcessBuilder = ProcessBuilder( "docker", "run", "--rm", "-v", + "${Paths.get(propertiesConfiguration.mountedVolume).toAbsolutePath()}" + + "/$ortFolder:/ortProject", + "-v", "${Paths.get(propertiesConfiguration.mountedVolume).toAbsolutePath()}" + "/$downloadFolder:/downloadedProject", "leanixacrpublic.azurecr.io/ort", @@ -65,7 +71,7 @@ class OrtService( "-P", "ort.analyzer.allowDynamicVersions=true", "analyze", "-i", "/downloadedProject", - "-o", "/downloadedProject" + "-o", "/ortProject" ) setupOutput(projectUrl, "analyze", analyzeProcessBuilder) @@ -74,20 +80,22 @@ class OrtService( analyzeProcess.waitFor(propertiesConfiguration.analysisTimeout, TimeUnit.MINUTES) analyzeProcess.destroy() + + return ortFolder } - fun generateSbom(projectUrl: String, downloadFolder: String) { + fun generateSbom(projectUrl: String) { val generateSbomProcessBuilder = ProcessBuilder( "docker", "run", "--rm", "-v", "${Paths.get(propertiesConfiguration.mountedVolume).toAbsolutePath()}" + - "/$downloadFolder:/downloadedProject", + "/${projectUrl.substringAfterLast("/")}_ORT_produced_files:/ortProject", "leanixacrpublic.azurecr.io/ort", loggingParameter(), "report", "-f", "CycloneDX", - "-i", "/downloadedProject/analyzer-result.yml", - "-o", "/downloadedProject", + "-i", "/ortProject/analyzer-result.yml", + "-o", "/ortProject", "-O", "CycloneDx=output.file.formats=json", "-O", "CycloneDx=schema.version=1.4" ) @@ -109,7 +117,7 @@ class OrtService( if (propertiesConfiguration.devMode) { val repoFileName = Paths.get( "tempDir", - "${projectUrl.substringAfterLast("/")}_$phase.txt" + "${projectUrl.substringAfterLast("/")}_${phase}_log.txt" ).toFile() FileOutputStream(repoFileName) diff --git a/src/main/kotlin/net/leanix/vsm/sbomBooster/service/ProcessService.kt b/src/main/kotlin/net/leanix/vsm/sbomBooster/service/ProcessService.kt index a52be6f..c538c19 100644 --- a/src/main/kotlin/net/leanix/vsm/sbomBooster/service/ProcessService.kt +++ b/src/main/kotlin/net/leanix/vsm/sbomBooster/service/ProcessService.kt @@ -29,6 +29,7 @@ class ProcessService( } @Async + @Suppress("LongMethod") fun processRepository( propertiesConfiguration: PropertiesConfiguration, username: String, @@ -37,6 +38,7 @@ class ProcessService( ) { val startInstant = Instant.now() var downloadedFolder: String? = null + var ortFolder: String? = null if (username.isNotBlank()) { try { logger.info("Beginning to download repository with url: ${repository.cloneUrl}") @@ -50,15 +52,15 @@ class ProcessService( ) logger.info("Beginning to analyze repository with url: ${repository.cloneUrl}") - ortService.analyzeProject(repository.cloneUrl, downloadedFolder) + ortFolder = ortService.analyzeProject(repository.cloneUrl, downloadedFolder) logger.info( - "Finished analyzing repository with url: ${repository.cloneUrl} in temp folder $downloadedFolder" + "Finished analyzing repository with url: ${repository.cloneUrl} in temp folder $ortFolder" ) - ortService.generateSbom(repository.cloneUrl, downloadedFolder) + ortService.generateSbom(repository.cloneUrl) logger.info( "Finished generating SBOM file for repository with url: " + - "${repository.cloneUrl} in temp folder $downloadedFolder." + "${repository.cloneUrl} in temp folder $ortFolder." ) val accessToken = mtMService.getAccessToken( @@ -73,7 +75,7 @@ class ProcessService( vsmRegion, VsmDiscoveryItem( repository.cloneUrl, - downloadedFolder, + ortFolder, repository.sourceType, repository.sourceInstance, repository.name, @@ -83,10 +85,14 @@ class ProcessService( } catch (e: Exception) { logger.error(e.message) } finally { + logger.info("Beginning to delete folder $downloadedFolder.") + ortService.deleteDownloadedFolder(downloadedFolder) + logger.info("Finished deleting temp folder $downloadedFolder.") + if (!propertiesConfiguration.devMode) { - logger.info("Beginning to delete folder $downloadedFolder.") - ortService.deleteDownloadedFolder(downloadedFolder) - logger.info("Finished deleting temp folder $downloadedFolder.") + logger.info("Beginning to delete folder $ortFolder.") + ortService.deleteDownloadedFolder(ortFolder) + logger.info("Finished deleting temp folder $ortFolder.") } } } diff --git a/src/main/kotlin/net/leanix/vsm/sbomBooster/service/VsmDiscoveryService.kt b/src/main/kotlin/net/leanix/vsm/sbomBooster/service/VsmDiscoveryService.kt index 4caaa9c..7d7a2d2 100644 --- a/src/main/kotlin/net/leanix/vsm/sbomBooster/service/VsmDiscoveryService.kt +++ b/src/main/kotlin/net/leanix/vsm/sbomBooster/service/VsmDiscoveryService.kt @@ -48,13 +48,13 @@ class VsmDiscoveryService( val sbomFile: Resource = FileSystemResource( "${Paths.get("tempDir").toAbsolutePath()}" + - "/${discoveryItem.downloadedFolder}/bom.cyclonedx.json" + "/${discoveryItem.ortFolder}/bom.cyclonedx.json" ) val sbomByteArray = Files.readAllBytes( Paths.get( "tempDir", - discoveryItem.downloadedFolder, + discoveryItem.ortFolder, "bom.cyclonedx.json" ) )