Skip to content

Commit

Permalink
Merge pull request #53 from leanix/feature/CID-1866/separate-ort-file…
Browse files Browse the repository at this point in the history 
…s-in-dev-mode

CID-1866 Separate ORT produced files retention from repository files
  • Loading branch information
@geoandri
geoandri authored Aug 24, 2023
2 parents 81d49a6 + 7707528 commit d29a8d4
Show file tree
Hide file tree
Showing 5 changed files with 33 additions and 19 deletions.
2 changes: 1 addition & 1 deletion README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -93,7 +93,7 @@ The second `-v` param is the path to temporary folder that the `vsm-sbom-booster

`ANALYSIS_TIMEOUT`(optional): The timeout, in minutes, that is used to force kill container workers working on analyzing the repository. There are cases that we need to force kill containers with slow progress to free up resources. Default: 30

`DEV_MODE`(optional): This is a flag to enable/disable the dev mode. When enabled, all logs from ORT containers will be preserved in the temp folder location and the project folders will be retained. Additionally, the logging level for the ORT containers will be set to DEBUG. The processing of each repository consists of three separate phases (download, analyze and generate_sbom) that are facilitated by the ORT software. The produced logs are saved in the temp folder using the `<repository_name>_<phase>.txt` naming pattern.This is useful for debugging purposes. Default: false
`DEV_MODE`(optional): This is a flag to enable/disable the dev mode. When enabled, all logs from ORT containers will be preserved in the temp folder location and the ORT project folders will be retained. Additionally, the logging level for the ORT containers will be set to DEBUG. The processing of each repository consists of three separate phases (download, analyze and generate_sbom) that are facilitated by the ORT software. The produced logs are saved in the temp folder using the `<repository_name>_<phase>_log.txt` naming pattern.This is useful for debugging purposes. Default: false

#### LeanIX configs

Expand Down
2 changes: 1 addition & 1 deletion src/main/kotlin/net/leanix/vsm/sbomBooster/domain/VsmDiscoveryItem.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ package net.leanix.vsm.sbomBooster.domain

data class VsmDiscoveryItem(
val projectUrl: String,
val downloadedFolder: String,
val ortFolder: String,
val sourceType: String,
val sourceInstance: String,
val name: String,
Expand Down
22 changes: 15 additions & 7 deletions src/main/kotlin/net/leanix/vsm/sbomBooster/service/OrtService.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -54,18 +54,24 @@ class OrtService(
return downloadFolder
}

fun analyzeProject(projectUrl: String, downloadFolder: String) {
fun analyzeProject(projectUrl: String, downloadFolder: String): String {

val ortFolder = "${projectUrl.substringAfterLast("/")}_ORT_produced_files"

val analyzeProcessBuilder = ProcessBuilder(
"docker", "run", "--rm",
"-v",
"${Paths.get(propertiesConfiguration.mountedVolume).toAbsolutePath()}" +
"/$ortFolder:/ortProject",
"-v",
"${Paths.get(propertiesConfiguration.mountedVolume).toAbsolutePath()}" +
"/$downloadFolder:/downloadedProject",
"leanixacrpublic.azurecr.io/ort",
loggingParameter(),
"-P", "ort.analyzer.allowDynamicVersions=true",
"analyze",
"-i", "/downloadedProject",
"-o", "/downloadedProject"
"-o", "/ortProject"
)

setupOutput(projectUrl, "analyze", analyzeProcessBuilder)
Expand All @@ -74,20 +80,22 @@ class OrtService(

analyzeProcess.waitFor(propertiesConfiguration.analysisTimeout, TimeUnit.MINUTES)
analyzeProcess.destroy()

return ortFolder
}

fun generateSbom(projectUrl: String, downloadFolder: String) {
fun generateSbom(projectUrl: String) {
val generateSbomProcessBuilder = ProcessBuilder(
"docker", "run", "--rm",
"-v",
"${Paths.get(propertiesConfiguration.mountedVolume).toAbsolutePath()}" +
"/$downloadFolder:/downloadedProject",
"/${projectUrl.substringAfterLast("/")}_ORT_produced_files:/ortProject",
"leanixacrpublic.azurecr.io/ort",
loggingParameter(),
"report",
"-f", "CycloneDX",
"-i", "/downloadedProject/analyzer-result.yml",
"-o", "/downloadedProject",
"-i", "/ortProject/analyzer-result.yml",
"-o", "/ortProject",
"-O", "CycloneDx=output.file.formats=json",
"-O", "CycloneDx=schema.version=1.4"
)
Expand All @@ -109,7 +117,7 @@ class OrtService(
if (propertiesConfiguration.devMode) {
val repoFileName = Paths.get(
"tempDir",
"${projectUrl.substringAfterLast("/")}_$phase.txt"
"${projectUrl.substringAfterLast("/")}_${phase}_log.txt"
).toFile()

FileOutputStream(repoFileName)
Expand Down
22 changes: 14 additions & 8 deletions src/main/kotlin/net/leanix/vsm/sbomBooster/service/ProcessService.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@ class ProcessService(
}

@Async
@Suppress("LongMethod")
fun processRepository(
propertiesConfiguration: PropertiesConfiguration,
username: String,
Expand All @@ -37,6 +38,7 @@ class ProcessService(
) {
val startInstant = Instant.now()
var downloadedFolder: String? = null
var ortFolder: String? = null
if (username.isNotBlank()) {
try {
logger.info("Beginning to download repository with url: ${repository.cloneUrl}")
Expand All @@ -50,15 +52,15 @@ class ProcessService(
)

logger.info("Beginning to analyze repository with url: ${repository.cloneUrl}")
ortService.analyzeProject(repository.cloneUrl, downloadedFolder)
ortFolder = ortService.analyzeProject(repository.cloneUrl, downloadedFolder)
logger.info(
"Finished analyzing repository with url: ${repository.cloneUrl} in temp folder $downloadedFolder"
"Finished analyzing repository with url: ${repository.cloneUrl} in temp folder $ortFolder"
)

ortService.generateSbom(repository.cloneUrl, downloadedFolder)
ortService.generateSbom(repository.cloneUrl)
logger.info(
"Finished generating SBOM file for repository with url: " +
"${repository.cloneUrl} in temp folder $downloadedFolder."
"${repository.cloneUrl} in temp folder $ortFolder."
)

val accessToken = mtMService.getAccessToken(
Expand All @@ -73,7 +75,7 @@ class ProcessService(
vsmRegion,
VsmDiscoveryItem(
repository.cloneUrl,
downloadedFolder,
ortFolder,
repository.sourceType,
repository.sourceInstance,
repository.name,
Expand All @@ -83,10 +85,14 @@ class ProcessService(
} catch (e: Exception) {
logger.error(e.message)
} finally {
logger.info("Beginning to delete folder $downloadedFolder.")
ortService.deleteDownloadedFolder(downloadedFolder)
logger.info("Finished deleting temp folder $downloadedFolder.")

if (!propertiesConfiguration.devMode) {
logger.info("Beginning to delete folder $downloadedFolder.")
ortService.deleteDownloadedFolder(downloadedFolder)
logger.info("Finished deleting temp folder $downloadedFolder.")
logger.info("Beginning to delete folder $ortFolder.")
ortService.deleteDownloadedFolder(ortFolder)
logger.info("Finished deleting temp folder $ortFolder.")
}
}
}
Expand Down
4 changes: 2 additions & 2 deletions src/main/kotlin/net/leanix/vsm/sbomBooster/service/VsmDiscoveryService.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -48,13 +48,13 @@ class VsmDiscoveryService(

val sbomFile: Resource = FileSystemResource(
"${Paths.get("tempDir").toAbsolutePath()}" +
"/${discoveryItem.downloadedFolder}/bom.cyclonedx.json"
"/${discoveryItem.ortFolder}/bom.cyclonedx.json"
)

val sbomByteArray = Files.readAllBytes(
Paths.get(
"tempDir",
discoveryItem.downloadedFolder,
discoveryItem.ortFolder,
"bom.cyclonedx.json"
)
)
Expand Down

0 comments on commit d29a8d4

Please sign in to comment.