From 35c66f3e34ab4c06770bf085ac5572a1157e1480 Mon Sep 17 00:00:00 2001 From: Yahav Itzhak Date: Wed, 20 Sep 2023 09:38:49 +0300 Subject: [PATCH 01/16] Transfer - Send chunks if they are bigger than GiB (#966) --- .../commands/transferfiles/api/types.go | 19 +++++++ .../commands/transferfiles/api/types_test.go | 51 +++++++++++++++++++ .../commands/transferfiles/fulltransfer.go | 2 +- .../commands/transferfiles/transfer.go | 1 - artifactory/commands/transferfiles/utils.go | 2 +- 5 files changed, 72 insertions(+), 3 deletions(-) create mode 100644 artifactory/commands/transferfiles/api/types_test.go diff --git a/artifactory/commands/transferfiles/api/types.go b/artifactory/commands/transferfiles/api/types.go index b9f0d8768..9cab62860 100644 --- a/artifactory/commands/transferfiles/api/types.go +++ b/artifactory/commands/transferfiles/api/types.go @@ -23,6 +23,10 @@ const ( Phase1 int = 0 Phase2 int = 1 Phase3 int = 2 + + maxFilesInChunk = 16 + // 1 GiB + maxBytesInChunk = 1 << 30 ) type TargetAuth struct { @@ -101,3 +105,18 @@ func (uc *UploadChunk) AppendUploadCandidateIfNeeded(file FileRepresentation, bu } uc.UploadCandidates = append(uc.UploadCandidates, file) } + +// Return true if the chunk contains at least 16 files or at least 1GiB in total +func (uc *UploadChunk) IsChunkFull() bool { + if len(uc.UploadCandidates) >= maxFilesInChunk { + return true + } + var totalSize int64 = 0 + for _, uploadCandidate := range uc.UploadCandidates { + totalSize += uploadCandidate.Size + if totalSize > maxBytesInChunk { + return true + } + } + return false +} diff --git a/artifactory/commands/transferfiles/api/types_test.go b/artifactory/commands/transferfiles/api/types_test.go new file mode 100644 index 000000000..c2ef3b207 --- /dev/null +++ b/artifactory/commands/transferfiles/api/types_test.go @@ -0,0 +1,51 @@ +package api + +import ( + "fmt" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestAppendUploadCandidateIfNeeded(t *testing.T) { + uploadChunk := &UploadChunk{} + + // Regular file + uploadChunk.AppendUploadCandidateIfNeeded(FileRepresentation{Name: "regular-file"}, false) + assert.Len(t, uploadChunk.UploadCandidates, 1) + + // Build info + uploadChunk.AppendUploadCandidateIfNeeded(FileRepresentation{Name: "build-info.json"}, true) + assert.Len(t, uploadChunk.UploadCandidates, 2) + + // Directory in build info - should be skipped + uploadChunk.AppendUploadCandidateIfNeeded(FileRepresentation{}, true) + assert.Len(t, uploadChunk.UploadCandidates, 2) +} + +var isChunkFullCases = []struct { + files []FileRepresentation + isFull bool +}{ + {[]FileRepresentation{}, false}, + {[]FileRepresentation{{Name: "slim-jim", Size: 10737418}}, false}, + {[]FileRepresentation{{Name: "fat-vinny", Size: 1073741825}}, true}, +} + +func TestIsChunkFull(t *testing.T) { + for _, testCase := range isChunkFullCases { + t.Run("", func(t *testing.T) { + uploadChunk := &UploadChunk{UploadCandidates: testCase.files} + assert.Equal(t, testCase.isFull, uploadChunk.IsChunkFull()) + }) + } +} + +func TestIsChunkFullNumberOfFiles(t *testing.T) { + uploadChunk := &UploadChunk{} + for i := 0; i < maxFilesInChunk; i++ { + assert.False(t, uploadChunk.IsChunkFull()) + uploadChunk.AppendUploadCandidateIfNeeded(FileRepresentation{Name: fmt.Sprintf("%d", i)}, false) + } + assert.True(t, uploadChunk.IsChunkFull()) +} diff --git a/artifactory/commands/transferfiles/fulltransfer.go b/artifactory/commands/transferfiles/fulltransfer.go index 3229982d9..4e90d7564 100644 --- a/artifactory/commands/transferfiles/fulltransfer.go +++ b/artifactory/commands/transferfiles/fulltransfer.go @@ -239,7 +239,7 @@ func (m *fullTransferPhase) handleFoundFile(pcWrapper producerConsumerWrapper, return } curUploadChunk.AppendUploadCandidateIfNeeded(file, m.buildInfoRepo) - if len(curUploadChunk.UploadCandidates) == uploadChunkSize { + if curUploadChunk.IsChunkFull() { _, err = pcWrapper.chunkUploaderProducerConsumer.AddTaskWithError(uploadChunkWhenPossibleHandler(&m.phaseBase, *curUploadChunk, uploadChunkChan, errorsChannelMng), pcWrapper.errorsQueue.AddError) if err != nil { return diff --git a/artifactory/commands/transferfiles/transfer.go b/artifactory/commands/transferfiles/transfer.go index fb45a01dc..218a9fb28 100644 --- a/artifactory/commands/transferfiles/transfer.go +++ b/artifactory/commands/transferfiles/transfer.go @@ -28,7 +28,6 @@ import ( ) const ( - uploadChunkSize = 16 // Size of the channel where the transfer's go routines write the transfer errors fileWritersChannelSize = 500000 retries = 600 diff --git a/artifactory/commands/transferfiles/utils.go b/artifactory/commands/transferfiles/utils.go index 668bebc1d..31fcd5045 100644 --- a/artifactory/commands/transferfiles/utils.go +++ b/artifactory/commands/transferfiles/utils.go @@ -419,7 +419,7 @@ func uploadByChunks(files []api.FileRepresentation, uploadTokensChan chan Upload continue } curUploadChunk.AppendUploadCandidateIfNeeded(file, base.buildInfoRepo) - if len(curUploadChunk.UploadCandidates) == uploadChunkSize { + if curUploadChunk.IsChunkFull() { _, err = pcWrapper.chunkUploaderProducerConsumer.AddTaskWithError(uploadChunkWhenPossibleHandler(&base, curUploadChunk, uploadTokensChan, errorsChannelMng), pcWrapper.errorsQueue.AddError) if err != nil { return From fd8b279f17af43692dd67b0047ac51edb0e4de8c Mon Sep 17 00:00:00 2001 From: Assaf Attias <49212512+attiasas@users.noreply.github.com> Date: Wed, 20 Sep 2023 16:08:27 +0300 Subject: [PATCH 02/16] Show findings in Sast (#970) --- xray/utils/resultstable.go | 1 + 1 file changed, 1 insertion(+) diff --git a/xray/utils/resultstable.go b/xray/utils/resultstable.go index ee04b677c..27fc58ccd 100644 --- a/xray/utils/resultstable.go +++ b/xray/utils/resultstable.go @@ -420,6 +420,7 @@ func prepareSast(sasts []*sarif.Run, isTable bool) []formats.SourceCodeRow { formats.SourceCodeRow{ SeverityDetails: formats.SeverityDetails{Severity: currSeverity.printableTitle(isTable), SeverityNumValue: currSeverity.NumValue()}, ScannerDescription: scannerDescription, + Finding: GetResultMsgText(sastResult), Location: formats.Location{ File: GetRelativeLocationFileName(location, sastRun.Invocations), StartLine: GetLocationStartLine(location), From 0414290c7deba247c86d9d969d5c35997177400c Mon Sep 17 00:00:00 2001 From: Assaf Attias <49212512+attiasas@users.noreply.github.com> Date: Thu, 21 Sep 2023 14:34:52 +0300 Subject: [PATCH 03/16] Add tests for audit (#965) --- xray/commands/audit/jas/common_test.go | 90 +++ xray/commands/audit/jas/sast/sastscanner.go | 34 +- .../audit/jas/sast/sastscanner_test.go | 92 ++- .../audit/jas/secrets/secretsscanner.go | 3 +- xray/utils/analyzermanager_test.go | 117 ---- xray/utils/resultstable_test.go | 370 +++++++++- xray/utils/sarifutils.go | 108 +-- xray/utils/sarifutils_test.go | 656 +++++++++++++++++- 8 files changed, 1250 insertions(+), 220 deletions(-) create mode 100644 xray/commands/audit/jas/common_test.go diff --git a/xray/commands/audit/jas/common_test.go b/xray/commands/audit/jas/common_test.go new file mode 100644 index 000000000..5466039d8 --- /dev/null +++ b/xray/commands/audit/jas/common_test.go @@ -0,0 +1,90 @@ +package jas + +import ( + "testing" + + "github.com/jfrog/jfrog-cli-core/v2/xray/utils" + "github.com/owenrumney/go-sarif/v2/sarif" + "github.com/stretchr/testify/assert" +) + +func TestExcludeSuppressResults(t *testing.T) { + tests := []struct { + name string + sarifResults []*sarif.Result + expectedOutput []*sarif.Result + }{ + { + sarifResults: []*sarif.Result{ + utils.CreateResultWithOneLocation("", 0, 0, 0, 0, "snippet1", "ruleId1", "level1"), + utils.CreateResultWithOneLocation("", 0, 0, 0, 0, "snippet2", "ruleId2", "level2"), + }, + expectedOutput: []*sarif.Result{ + utils.CreateResultWithOneLocation("", 0, 0, 0, 0, "snippet1", "ruleId1", "level1"), + utils.CreateResultWithOneLocation("", 0, 0, 0, 0, "snippet2", "ruleId2", "level2"), + }, + }, + { + sarifResults: []*sarif.Result{ + utils.CreateResultWithOneLocation("", 0, 0, 0, 0, "snippet1", "ruleId1", "level1").WithSuppression([]*sarif.Suppression{sarif.NewSuppression("")}), + utils.CreateResultWithOneLocation("", 0, 0, 0, 0, "snippet2", "ruleId2", "level2"), + }, + expectedOutput: []*sarif.Result{ + utils.CreateResultWithOneLocation("", 0, 0, 0, 0, "snippet2", "ruleId2", "level2"), + }, + }, + { + sarifResults: []*sarif.Result{ + utils.CreateResultWithOneLocation("", 0, 0, 0, 0, "snippet1", "ruleId1", "level1").WithSuppression([]*sarif.Suppression{sarif.NewSuppression("")}), + utils.CreateResultWithOneLocation("", 0, 0, 0, 0, "snippet2", "ruleId2", "level2").WithSuppression([]*sarif.Suppression{sarif.NewSuppression("")}), + }, + expectedOutput: []*sarif.Result{}, + }, + } + + for _, test := range tests { + assert.Equal(t, test.expectedOutput, excludeSuppressResults(test.sarifResults)) + } +} + +func TestAddScoreToRunRules(t *testing.T) { + + tests := []struct { + name string + sarifRun *sarif.Run + expectedOutput []*sarif.ReportingDescriptor + }{ + { + sarifRun: utils.CreateRunWithDummyResults( + utils.CreateResultWithOneLocation("file1", 0, 0, 0, 0, "snippet", "rule1", "info"), + utils.CreateResultWithOneLocation("file2", 0, 0, 0, 0, "snippet", "rule1", "info"), + utils.CreateResultWithOneLocation("file", 0, 0, 0, 0, "snippet", "rule2", "warning"), + ), + expectedOutput: []*sarif.ReportingDescriptor{ + sarif.NewRule("rule1").WithProperties(sarif.Properties{"security-severity": "6.9"}), + sarif.NewRule("rule2").WithProperties(sarif.Properties{"security-severity": "6.9"}), + }, + }, + { + sarifRun: utils.CreateRunWithDummyResults( + utils.CreateResultWithOneLocation("file", 0, 0, 0, 0, "snippet", "rule1", "none"), + utils.CreateResultWithOneLocation("file", 0, 0, 0, 0, "snippet", "rule2", "note"), + utils.CreateResultWithOneLocation("file", 0, 0, 0, 0, "snippet", "rule3", "info"), + utils.CreateResultWithOneLocation("file", 0, 0, 0, 0, "snippet", "rule4", "warning"), + utils.CreateResultWithOneLocation("file", 0, 0, 0, 0, "snippet", "rule5", "error"), + ), + expectedOutput: []*sarif.ReportingDescriptor{ + sarif.NewRule("rule1").WithProperties(sarif.Properties{"security-severity": "0.0"}), + sarif.NewRule("rule2").WithProperties(sarif.Properties{"security-severity": "3.9"}), + sarif.NewRule("rule3").WithProperties(sarif.Properties{"security-severity": "6.9"}), + sarif.NewRule("rule4").WithProperties(sarif.Properties{"security-severity": "6.9"}), + sarif.NewRule("rule5").WithProperties(sarif.Properties{"security-severity": "8.9"}), + }, + }, + } + + for _, test := range tests { + addScoreToRunRules(test.sarifRun) + assert.Equal(t, test.expectedOutput, test.sarifRun.Tool.Driver.Rules) + } +} diff --git a/xray/commands/audit/jas/sast/sastscanner.go b/xray/commands/audit/jas/sast/sastscanner.go index 35211396f..d4402cd68 100644 --- a/xray/commands/audit/jas/sast/sastscanner.go +++ b/xray/commands/audit/jas/sast/sastscanner.go @@ -1,6 +1,8 @@ package sast import ( + "fmt" + "github.com/jfrog/jfrog-cli-core/v2/xray/commands/audit/jas" "github.com/jfrog/jfrog-cli-core/v2/xray/utils" "github.com/jfrog/jfrog-client-go/utils/log" @@ -47,7 +49,8 @@ func (ssm *SastScanManager) Run(wd string) (err error) { if err != nil { return } - ssm.sastScannerResults = append(ssm.sastScannerResults, groupResultsByLocation(workingDirRuns)...) + groupResultsByLocation(workingDirRuns) + ssm.sastScannerResults = append(ssm.sastScannerResults, workingDirRuns...) return } @@ -58,7 +61,7 @@ func (ssm *SastScanManager) runAnalyzerManager(wd string) error { // In the Sast scanner, there can be multiple results with the same location. // The only difference is that their CodeFlow values are different. // We combine those under the same result location value -func groupResultsByLocation(sarifRuns []*sarif.Run) []*sarif.Run { +func groupResultsByLocation(sarifRuns []*sarif.Run) { for _, sastRun := range sarifRuns { locationToResult := map[string]*sarif.Result{} for _, sastResult := range sastRun.Results { @@ -71,25 +74,28 @@ func groupResultsByLocation(sarifRuns []*sarif.Run) []*sarif.Run { } sastRun.Results = maps.Values(locationToResult) } - return sarifRuns } -// In Sast there is only one location for each result -func getResultFileName(result *sarif.Result) string { - if len(result.Locations) > 0 { - return utils.GetLocationFileName(result.Locations[0]) +func getResultLocationStr(result *sarif.Result) string { + if len(result.Locations) == 0 { + return "" } - return "" + location := result.Locations[0] + return fmt.Sprintf("%s%d%d%d%d", + utils.GetLocationFileName(location), + utils.GetLocationStartLine(location), + utils.GetLocationStartColumn(location), + utils.GetLocationEndLine(location), + utils.GetLocationEndColumn(location)) } -// In Sast there is only one location for each result -func getResultStartLocationInFile(result *sarif.Result) string { - if len(result.Locations) > 0 { - return utils.GetStartLocationInFile(result.Locations[0]) +func getResultRuleId(result *sarif.Result) string { + if result.RuleID == nil { + return "" } - return "" + return *result.RuleID } func getResultId(result *sarif.Result) string { - return getResultFileName(result) + getResultStartLocationInFile(result) + utils.GetResultSeverity(result) + utils.GetResultMsgText(result) + return getResultRuleId(result) + utils.GetResultSeverity(result) + utils.GetResultMsgText(result) + getResultLocationStr(result) } diff --git a/xray/commands/audit/jas/sast/sastscanner_test.go b/xray/commands/audit/jas/sast/sastscanner_test.go index 66c423403..6ed1980f3 100644 --- a/xray/commands/audit/jas/sast/sastscanner_test.go +++ b/xray/commands/audit/jas/sast/sastscanner_test.go @@ -5,6 +5,8 @@ import ( "testing" "github.com/jfrog/jfrog-cli-core/v2/xray/commands/audit/jas" + "github.com/jfrog/jfrog-cli-core/v2/xray/utils" + "github.com/owenrumney/go-sarif/v2/sarif" "github.com/stretchr/testify/assert" ) @@ -40,7 +42,7 @@ func TestSastParseResults_EmptyResults(t *testing.T) { if assert.NoError(t, err) && assert.NotNil(t, sastScanManager.sastScannerResults) { assert.Len(t, sastScanManager.sastScannerResults, 1) assert.Empty(t, sastScanManager.sastScannerResults[0].Results) - sastScanManager.sastScannerResults = groupResultsByLocation(sastScanManager.sastScannerResults) + groupResultsByLocation(sastScanManager.sastScannerResults) assert.Len(t, sastScanManager.sastScannerResults, 1) assert.Empty(t, sastScanManager.sastScannerResults[0].Results) } @@ -61,8 +63,94 @@ func TestSastParseResults_ResultsContainIacViolations(t *testing.T) { if assert.NoError(t, err) && assert.NotNil(t, sastScanManager.sastScannerResults) { assert.Len(t, sastScanManager.sastScannerResults, 1) assert.NotEmpty(t, sastScanManager.sastScannerResults[0].Results) - sastScanManager.sastScannerResults = groupResultsByLocation(sastScanManager.sastScannerResults) + groupResultsByLocation(sastScanManager.sastScannerResults) // File has 4 results, 2 of them at the same location different codeFlow assert.Len(t, sastScanManager.sastScannerResults[0].Results, 3) } } + +func TestGroupResultsByLocation(t *testing.T) { + tests := []struct { + run *sarif.Run + expectedOutput *sarif.Run + }{ + { + run: utils.CreateRunWithDummyResults(), + expectedOutput: utils.CreateRunWithDummyResults(), + }, + { + // No similar groups at all + run: utils.CreateRunWithDummyResults( + utils.CreateResultWithOneLocation("file", 1, 2, 3, 4, "snippet", "rule1", "info"), + utils.CreateResultWithOneLocation("file", 1, 2, 3, 4, "snippet", "rule1", "note"), + utils.CreateResultWithOneLocation("file", 5, 6, 7, 8, "snippet", "rule1", "info"), + utils.CreateResultWithOneLocation("file2", 1, 2, 3, 4, "snippet", "rule1", "info").WithCodeFlows([]*sarif.CodeFlow{ + utils.CreateCodeFlow(utils.CreateThreadFlow( + utils.CreateLocation("other", 0, 0, 0, 0, "other-snippet"), + utils.CreateLocation("file2", 1, 2, 3, 4, "snippet"), + )), + }), + utils.CreateResultWithOneLocation("file2", 1, 2, 3, 4, "snippet", "rule2", "info").WithCodeFlows([]*sarif.CodeFlow{ + utils.CreateCodeFlow(utils.CreateThreadFlow( + utils.CreateLocation("other2", 1, 1, 1, 1, "other-snippet2"), + utils.CreateLocation("file2", 1, 2, 3, 4, "snippet"), + )), + }), + ), + expectedOutput: utils.CreateRunWithDummyResults( + utils.CreateResultWithOneLocation("file", 1, 2, 3, 4, "snippet", "rule1", "info"), + utils.CreateResultWithOneLocation("file", 1, 2, 3, 4, "snippet", "rule1", "note"), + utils.CreateResultWithOneLocation("file", 5, 6, 7, 8, "snippet", "rule1", "info"), + utils.CreateResultWithOneLocation("file2", 1, 2, 3, 4, "snippet", "rule1", "info").WithCodeFlows([]*sarif.CodeFlow{ + utils.CreateCodeFlow(utils.CreateThreadFlow( + utils.CreateLocation("other", 0, 0, 0, 0, "other-snippet"), + utils.CreateLocation("file2", 1, 2, 3, 4, "snippet"), + )), + }), + utils.CreateResultWithOneLocation("file2", 1, 2, 3, 4, "snippet", "rule2", "info").WithCodeFlows([]*sarif.CodeFlow{ + utils.CreateCodeFlow(utils.CreateThreadFlow( + utils.CreateLocation("other2", 1, 1, 1, 1, "other-snippet2"), + utils.CreateLocation("file2", 1, 2, 3, 4, "snippet"), + )), + }), + ), + }, + { + // With similar groups + run: utils.CreateRunWithDummyResults( + utils.CreateResultWithOneLocation("file", 1, 2, 3, 4, "snippet", "rule1", "info").WithCodeFlows([]*sarif.CodeFlow{ + utils.CreateCodeFlow(utils.CreateThreadFlow( + utils.CreateLocation("other", 0, 0, 0, 0, "other-snippet"), + utils.CreateLocation("file", 1, 2, 3, 4, "snippet"), + )), + }), + utils.CreateResultWithOneLocation("file", 1, 2, 3, 4, "snippet", "rule1", "info").WithCodeFlows([]*sarif.CodeFlow{ + utils.CreateCodeFlow(utils.CreateThreadFlow( + utils.CreateLocation("other2", 1, 1, 1, 1, "other-snippet"), + utils.CreateLocation("file", 1, 2, 3, 4, "snippet"), + )), + }), + utils.CreateResultWithOneLocation("file", 5, 6, 7, 8, "snippet", "rule1", "info"), + utils.CreateResultWithOneLocation("file", 1, 2, 3, 4, "snippet", "rule1", "info"), + ), + expectedOutput: utils.CreateRunWithDummyResults( + utils.CreateResultWithOneLocation("file", 1, 2, 3, 4, "snippet", "rule1", "info").WithCodeFlows([]*sarif.CodeFlow{ + utils.CreateCodeFlow(utils.CreateThreadFlow( + utils.CreateLocation("other", 0, 0, 0, 0, "other-snippet"), + utils.CreateLocation("file", 1, 2, 3, 4, "snippet"), + )), + utils.CreateCodeFlow(utils.CreateThreadFlow( + utils.CreateLocation("other2", 1, 1, 1, 1, "other-snippet"), + utils.CreateLocation("file", 1, 2, 3, 4, "snippet"), + )), + }), + utils.CreateResultWithOneLocation("file", 5, 6, 7, 8, "snippet", "rule1", "info"), + ), + }, + } + + for _, test := range tests { + groupResultsByLocation([]*sarif.Run{test.run}) + assert.ElementsMatch(t, test.expectedOutput.Results, test.run.Results) + } +} diff --git a/xray/commands/audit/jas/secrets/secretsscanner.go b/xray/commands/audit/jas/secrets/secretsscanner.go index cf5df05f8..6f0b985ff 100644 --- a/xray/commands/audit/jas/secrets/secretsscanner.go +++ b/xray/commands/audit/jas/secrets/secretsscanner.go @@ -105,8 +105,7 @@ func processSecretScanRuns(sarifRuns []*sarif.Run) []*sarif.Run { // Hide discovered secrets value for _, secretResult := range secretRun.Results { for _, location := range secretResult.Locations { - secret := utils.GetLocationSnippetPointer(location) - utils.SetLocationSnippet(location, maskSecret(*secret)) + utils.SetLocationSnippet(location, maskSecret(utils.GetLocationSnippet(location))) } } } diff --git a/xray/utils/analyzermanager_test.go b/xray/utils/analyzermanager_test.go index c8637a7e2..d493bfb4b 100644 --- a/xray/utils/analyzermanager_test.go +++ b/xray/utils/analyzermanager_test.go @@ -5,126 +5,9 @@ import ( "fmt" "testing" - "github.com/owenrumney/go-sarif/v2/sarif" "github.com/stretchr/testify/assert" ) -func TestGetResultFileName(t *testing.T) { - fileNameValue := "fileNameValue" - tests := []struct { - result *sarif.Result - expectedOutput string - }{ - {result: &sarif.Result{ - Locations: []*sarif.Location{ - {PhysicalLocation: &sarif.PhysicalLocation{ArtifactLocation: &sarif.ArtifactLocation{URI: nil}}}, - }}, - expectedOutput: ""}, - {result: &sarif.Result{ - Locations: []*sarif.Location{ - {PhysicalLocation: &sarif.PhysicalLocation{ArtifactLocation: &sarif.ArtifactLocation{URI: &fileNameValue}}}, - }}, - expectedOutput: fileNameValue}, - } - - for _, test := range tests { - assert.Equal(t, test.expectedOutput, GetLocationFileName(test.result.Locations[0])) - } - -} - -func TestGetResultLocationInFile(t *testing.T) { - startLine := 19 - startColumn := 25 - - tests := []struct { - result *sarif.Result - expectedOutput string - }{ - {result: &sarif.Result{Locations: []*sarif.Location{ - {PhysicalLocation: &sarif.PhysicalLocation{Region: &sarif.Region{ - StartLine: &startLine, - StartColumn: &startColumn, - }}}}}, - expectedOutput: "19:25"}, - {result: &sarif.Result{Locations: []*sarif.Location{ - {PhysicalLocation: &sarif.PhysicalLocation{Region: &sarif.Region{ - StartLine: nil, - StartColumn: &startColumn, - }}}}}, - expectedOutput: ""}, - {result: &sarif.Result{Locations: []*sarif.Location{ - {PhysicalLocation: &sarif.PhysicalLocation{Region: &sarif.Region{ - StartLine: &startLine, - StartColumn: nil, - }}}}}, - expectedOutput: ""}, - {result: &sarif.Result{Locations: []*sarif.Location{ - {PhysicalLocation: &sarif.PhysicalLocation{Region: &sarif.Region{ - StartLine: nil, - StartColumn: nil, - }}}}}, - expectedOutput: ""}, - } - - for _, test := range tests { - assert.Equal(t, test.expectedOutput, GetStartLocationInFile(test.result.Locations[0])) - } -} - -func TestExtractRelativePath(t *testing.T) { - tests := []struct { - secretPath string - projectPath string - expectedResult string - }{ - {secretPath: "file:///Users/user/Desktop/secrets_scanner/tests/req.nodejs/file.js", - projectPath: "Users/user/Desktop/secrets_scanner/", expectedResult: "tests/req.nodejs/file.js"}, - {secretPath: "invalidSecretPath", - projectPath: "Users/user/Desktop/secrets_scanner/", expectedResult: "invalidSecretPath"}, - {secretPath: "", - projectPath: "Users/user/Desktop/secrets_scanner/", expectedResult: ""}, - {secretPath: "file:///Users/user/Desktop/secrets_scanner/tests/req.nodejs/file.js", - projectPath: "invalidProjectPath", expectedResult: "Users/user/Desktop/secrets_scanner/tests/req.nodejs/file.js"}, - {secretPath: "file:///private/Users/user/Desktop/secrets_scanner/tests/req.nodejs/file.js", - projectPath: "invalidProjectPath", expectedResult: "Users/user/Desktop/secrets_scanner/tests/req.nodejs/file.js"}, - } - - for _, test := range tests { - assert.Equal(t, test.expectedResult, ExtractRelativePath(test.secretPath, test.projectPath)) - } -} - -func TestGetResultSeverity(t *testing.T) { - levelValueHigh := string(errorLevel) - levelValueMedium := string(warningLevel) - levelValueMedium2 := string(infoLevel) - levelValueLow := string(noteLevel) - levelValueUnknown := string(noneLevel) - - tests := []struct { - result *sarif.Result - expectedSeverity string - }{ - {result: &sarif.Result{}, - expectedSeverity: "Medium"}, - {result: &sarif.Result{Level: &levelValueHigh}, - expectedSeverity: "High"}, - {result: &sarif.Result{Level: &levelValueMedium}, - expectedSeverity: "Medium"}, - {result: &sarif.Result{Level: &levelValueMedium2}, - expectedSeverity: "Medium"}, - {result: &sarif.Result{Level: &levelValueLow}, - expectedSeverity: "Low"}, - {result: &sarif.Result{Level: &levelValueUnknown}, - expectedSeverity: "Unknown"}, - } - - for _, test := range tests { - assert.Equal(t, test.expectedSeverity, GetResultSeverity(test.result)) - } -} - func TestScanTypeErrorMsg(t *testing.T) { tests := []struct { scanner JasScanType diff --git a/xray/utils/resultstable_test.go b/xray/utils/resultstable_test.go index 66d72bfb3..3606c1b8a 100644 --- a/xray/utils/resultstable_test.go +++ b/xray/utils/resultstable_test.go @@ -439,9 +439,9 @@ func TestGetApplicableCveValue(t *testing.T) { { scanResults: &ExtendedScanResults{ ApplicabilityScanResults: []*sarif.Run{ - getRunWithDummyResults( - getDummyResultWithOneLocation("fileName1", 0, 1, "snippet1", "applic_testCve1", "info"), - getDummyPassingResult("applic_testCve2"), + CreateRunWithDummyResults( + CreateResultWithOneLocation("fileName1", 0, 1, 0, 0, "snippet1", "applic_testCve1", "info"), + CreateDummyPassingResult("applic_testCve2"), ), }, EntitledForJas: true, @@ -453,9 +453,9 @@ func TestGetApplicableCveValue(t *testing.T) { { scanResults: &ExtendedScanResults{ ApplicabilityScanResults: []*sarif.Run{ - getRunWithDummyResults( - getDummyPassingResult("applic_testCve1"), - getDummyResultWithOneLocation("fileName2", 1, 0, "snippet2", "applic_testCve2", "warning"), + CreateRunWithDummyResults( + CreateDummyPassingResult("applic_testCve1"), + CreateResultWithOneLocation("fileName2", 1, 0, 0, 0, "snippet2", "applic_testCve2", "warning"), ), }, EntitledForJas: true, @@ -467,9 +467,9 @@ func TestGetApplicableCveValue(t *testing.T) { { scanResults: &ExtendedScanResults{ ApplicabilityScanResults: []*sarif.Run{ - getRunWithDummyResults( - getDummyPassingResult("applic_testCve1"), - getDummyResultWithOneLocation("fileName3", 0, 1, "snippet3", "applic_testCve2", "info"), + CreateRunWithDummyResults( + CreateDummyPassingResult("applic_testCve1"), + CreateResultWithOneLocation("fileName3", 0, 1, 0, 0, "snippet3", "applic_testCve2", "info"), ), }, EntitledForJas: true, @@ -481,9 +481,9 @@ func TestGetApplicableCveValue(t *testing.T) { { scanResults: &ExtendedScanResults{ ApplicabilityScanResults: []*sarif.Run{ - getRunWithDummyResults( - getDummyPassingResult("applic_testCve1"), - getDummyPassingResult("applic_testCve2"), + CreateRunWithDummyResults( + CreateDummyPassingResult("applic_testCve1"), + CreateDummyPassingResult("applic_testCve2"), ), }, EntitledForJas: true, @@ -495,9 +495,9 @@ func TestGetApplicableCveValue(t *testing.T) { { scanResults: &ExtendedScanResults{ ApplicabilityScanResults: []*sarif.Run{ - getRunWithDummyResults( - getDummyPassingResult("applic_testCve1"), - getDummyResultWithOneLocation("fileName4", 1, 0, "snippet", "applic_testCve2", "warning"), + CreateRunWithDummyResults( + CreateDummyPassingResult("applic_testCve1"), + CreateResultWithOneLocation("fileName4", 1, 0, 0, 0, "snippet", "applic_testCve2", "warning"), ), }, EntitledForJas: true, @@ -509,7 +509,7 @@ func TestGetApplicableCveValue(t *testing.T) { { scanResults: &ExtendedScanResults{ ApplicabilityScanResults: []*sarif.Run{ - getRunWithDummyResults(getDummyPassingResult("applic_testCve1")), + CreateRunWithDummyResults(CreateDummyPassingResult("applic_testCve1")), }, EntitledForJas: true}, cves: []services.Cve{{Id: "testCve1"}, {Id: "testCve2"}}, @@ -706,6 +706,344 @@ func TestShouldDisqualifyEvidence(t *testing.T) { } } +func TestPrepareIac(t *testing.T) { + testCases := []struct { + name string + input []*sarif.Run + expectedOutput []formats.SourceCodeRow + }{ + { + name: "No Iac run", + input: []*sarif.Run{}, + expectedOutput: []formats.SourceCodeRow{}, + }, + { + name: "Prepare Iac run - no results", + input: []*sarif.Run{ + CreateRunWithDummyResults(), + CreateRunWithDummyResults(), + CreateRunWithDummyResults(), + }, + expectedOutput: []formats.SourceCodeRow{}, + }, + { + name: "Prepare Iac run - with results", + input: []*sarif.Run{ + CreateRunWithDummyResults(), + CreateRunWithDummyResults( + CreateResultWithLocations("iac finding", "rule1", "info", + CreateLocation("file://wd/file", 1, 2, 3, 4, "snippet"), + CreateLocation("file://wd/file2", 5, 6, 7, 8, "other-snippet"), + ), + ).WithInvocations([]*sarif.Invocation{ + sarif.NewInvocation().WithWorkingDirectory(sarif.NewSimpleArtifactLocation("wd")), + }), + CreateRunWithDummyResults( + CreateResultWithLocations("other iac finding", "rule2", "error", + CreateLocation("file://wd2/file3", 1, 2, 3, 4, "snippet"), + ), + ).WithInvocations([]*sarif.Invocation{ + sarif.NewInvocation().WithWorkingDirectory(sarif.NewSimpleArtifactLocation("wd2")), + }), + }, + expectedOutput: []formats.SourceCodeRow{ + { + SeverityDetails: formats.SeverityDetails{ + Severity: "High", + SeverityNumValue: 13, + }, + Finding: "other iac finding", + Location: formats.Location{ + File: "file3", + StartLine: 1, + StartColumn: 2, + EndLine: 3, + EndColumn: 4, + Snippet: "snippet", + }, + }, + { + SeverityDetails: formats.SeverityDetails{ + Severity: "Medium", + SeverityNumValue: 11, + }, + Finding: "iac finding", + Location: formats.Location{ + File: "file", + StartLine: 1, + StartColumn: 2, + EndLine: 3, + EndColumn: 4, + Snippet: "snippet", + }, + }, + { + SeverityDetails: formats.SeverityDetails{ + Severity: "Medium", + SeverityNumValue: 11, + }, + Finding: "iac finding", + Location: formats.Location{ + File: "file2", + StartLine: 5, + StartColumn: 6, + EndLine: 7, + EndColumn: 8, + Snippet: "other-snippet", + }, + }, + }, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + assert.ElementsMatch(t, tc.expectedOutput, prepareIacs(tc.input, false)) + }) + } +} + +func TestPrepareSecrets(t *testing.T) { + testCases := []struct { + name string + input []*sarif.Run + expectedOutput []formats.SourceCodeRow + }{ + { + name: "No Secret run", + input: []*sarif.Run{}, + expectedOutput: []formats.SourceCodeRow{}, + }, + { + name: "Prepare Secret run - no results", + input: []*sarif.Run{ + CreateRunWithDummyResults(), + CreateRunWithDummyResults(), + CreateRunWithDummyResults(), + }, + expectedOutput: []formats.SourceCodeRow{}, + }, + { + name: "Prepare Secret run - with results", + input: []*sarif.Run{ + CreateRunWithDummyResults(), + CreateRunWithDummyResults( + CreateResultWithLocations("secret finding", "rule1", "info", + CreateLocation("file://wd/file", 1, 2, 3, 4, "some-secret-snippet"), + CreateLocation("file://wd/file2", 5, 6, 7, 8, "other-secret-snippet"), + ), + ).WithInvocations([]*sarif.Invocation{ + sarif.NewInvocation().WithWorkingDirectory(sarif.NewSimpleArtifactLocation("wd")), + }), + CreateRunWithDummyResults( + CreateResultWithLocations("other secret finding", "rule2", "note", + CreateLocation("file://wd2/file3", 1, 2, 3, 4, "some-secret-snippet"), + ), + ).WithInvocations([]*sarif.Invocation{ + sarif.NewInvocation().WithWorkingDirectory(sarif.NewSimpleArtifactLocation("wd2")), + }), + }, + expectedOutput: []formats.SourceCodeRow{ + { + SeverityDetails: formats.SeverityDetails{ + Severity: "Low", + SeverityNumValue: 9, + }, + Finding: "other secret finding", + Location: formats.Location{ + File: "file3", + StartLine: 1, + StartColumn: 2, + EndLine: 3, + EndColumn: 4, + Snippet: "some-secret-snippet", + }, + }, + { + SeverityDetails: formats.SeverityDetails{ + Severity: "Medium", + SeverityNumValue: 11, + }, + Finding: "secret finding", + Location: formats.Location{ + File: "file", + StartLine: 1, + StartColumn: 2, + EndLine: 3, + EndColumn: 4, + Snippet: "some-secret-snippet", + }, + }, + { + SeverityDetails: formats.SeverityDetails{ + Severity: "Medium", + SeverityNumValue: 11, + }, + Finding: "secret finding", + Location: formats.Location{ + File: "file2", + StartLine: 5, + StartColumn: 6, + EndLine: 7, + EndColumn: 8, + Snippet: "other-secret-snippet", + }, + }, + }, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + assert.ElementsMatch(t, tc.expectedOutput, prepareSecrets(tc.input, false)) + }) + } +} + +func TestPrepareSast(t *testing.T) { + testCases := []struct { + name string + input []*sarif.Run + expectedOutput []formats.SourceCodeRow + }{ + { + name: "No Sast run", + input: []*sarif.Run{}, + expectedOutput: []formats.SourceCodeRow{}, + }, + { + name: "Prepare Sast run - no results", + input: []*sarif.Run{ + CreateRunWithDummyResults(), + CreateRunWithDummyResults(), + CreateRunWithDummyResults(), + }, + expectedOutput: []formats.SourceCodeRow{}, + }, + { + name: "Prepare Sast run - with results", + input: []*sarif.Run{ + CreateRunWithDummyResults(), + CreateRunWithDummyResults( + CreateResultWithLocations("sast finding", "rule1", "info", + CreateLocation("file://wd/file", 1, 2, 3, 4, "snippet"), + CreateLocation("file://wd/file2", 5, 6, 7, 8, "other-snippet"), + ).WithCodeFlows([]*sarif.CodeFlow{ + CreateCodeFlow(CreateThreadFlow( + CreateLocation("file://wd/file2", 0, 2, 0, 2, "snippetA"), + CreateLocation("file://wd/file", 1, 2, 3, 4, "snippet"), + )), + CreateCodeFlow(CreateThreadFlow( + CreateLocation("file://wd/file4", 1, 0, 1, 8, "snippetB"), + CreateLocation("file://wd/file", 1, 2, 3, 4, "snippet"), + )), + }), + ).WithInvocations([]*sarif.Invocation{ + sarif.NewInvocation().WithWorkingDirectory(sarif.NewSimpleArtifactLocation("wd")), + }), + CreateRunWithDummyResults( + CreateResultWithLocations("other sast finding", "rule2", "error", + CreateLocation("file://wd2/file3", 1, 2, 3, 4, "snippet"), + ), + ).WithInvocations([]*sarif.Invocation{ + sarif.NewInvocation().WithWorkingDirectory(sarif.NewSimpleArtifactLocation("wd2")), + }), + }, + expectedOutput: []formats.SourceCodeRow{ + { + SeverityDetails: formats.SeverityDetails{ + Severity: "High", + SeverityNumValue: 13, + }, + Finding: "other sast finding", + Location: formats.Location{ + File: "file3", + StartLine: 1, + StartColumn: 2, + EndLine: 3, + EndColumn: 4, + Snippet: "snippet", + }, + }, + { + SeverityDetails: formats.SeverityDetails{ + Severity: "Medium", + SeverityNumValue: 11, + }, + Finding: "sast finding", + Location: formats.Location{ + File: "file", + StartLine: 1, + StartColumn: 2, + EndLine: 3, + EndColumn: 4, + Snippet: "snippet", + }, + CodeFlow: [][]formats.Location{ + { + { + File: "file2", + StartLine: 0, + StartColumn: 2, + EndLine: 0, + EndColumn: 2, + Snippet: "snippetA", + }, + { + File: "file", + StartLine: 1, + StartColumn: 2, + EndLine: 3, + EndColumn: 4, + Snippet: "snippet", + }, + }, + { + { + File: "file4", + StartLine: 1, + StartColumn: 0, + EndLine: 1, + EndColumn: 8, + Snippet: "snippetB", + }, + { + File: "file", + StartLine: 1, + StartColumn: 2, + EndLine: 3, + EndColumn: 4, + Snippet: "snippet", + }, + }, + }, + }, + { + SeverityDetails: formats.SeverityDetails{ + Severity: "Medium", + SeverityNumValue: 11, + }, + Finding: "sast finding", + Location: formats.Location{ + File: "file2", + StartLine: 5, + StartColumn: 6, + EndLine: 7, + EndColumn: 8, + Snippet: "other-snippet", + }, + }, + }, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + assert.ElementsMatch(t, tc.expectedOutput, prepareSast(tc.input, false)) + }) + } +} + func newBoolPtr(v bool) *bool { return &v } diff --git a/xray/utils/sarifutils.go b/xray/utils/sarifutils.go index 5b5d31ab2..d0a959bfe 100644 --- a/xray/utils/sarifutils.go +++ b/xray/utils/sarifutils.go @@ -3,7 +3,6 @@ package utils import ( "fmt" "path/filepath" - "strconv" "strings" "github.com/jfrog/jfrog-client-go/utils/errorutils" @@ -115,24 +114,6 @@ func GetResultsLocationCount(runs ...*sarif.Run) (count int) { return } -func GetLevelResultsLocationCount(run *sarif.Run, level SarifLevel) (count int) { - for _, result := range run.Results { - if level == SarifLevel(*result.Level) { - count += len(result.Locations) - } - } - return -} - -func GetResultsByRuleId(run *sarif.Run, ruleId string) (results []*sarif.Result) { - for _, result := range run.Results { - if *result.RuleID == ruleId { - results = append(results, result) - } - } - return -} - func GetResultMsgText(result *sarif.Result) string { if result.Message.Text != nil { return *result.Message.Text @@ -141,19 +122,11 @@ func GetResultMsgText(result *sarif.Result) string { } func GetLocationSnippet(location *sarif.Location) string { - snippet := GetLocationSnippetPointer(location) - if snippet == nil { - return "" - } - return *snippet -} - -func GetLocationSnippetPointer(location *sarif.Location) *string { region := getLocationRegion(location) if region != nil && region.Snippet != nil { - return region.Snippet.Text + return *region.Snippet.Text } - return nil + return "" } func SetLocationSnippet(location *sarif.Location, snippet string) { @@ -163,9 +136,8 @@ func SetLocationSnippet(location *sarif.Location, snippet string) { } func GetLocationFileName(location *sarif.Location) string { - filePath := location.PhysicalLocation.ArtifactLocation.URI - if filePath != nil { - return *filePath + if location != nil && location.PhysicalLocation != nil && location.PhysicalLocation.ArtifactLocation != nil && location.PhysicalLocation.ArtifactLocation.URI != nil { + return *location.PhysicalLocation.ArtifactLocation.URI } return "" } @@ -228,15 +200,6 @@ func GetLocationEndColumn(location *sarif.Location) int { return 0 } -func GetStartLocationInFile(location *sarif.Location) string { - startLine := location.PhysicalLocation.Region.StartLine - startColumn := location.PhysicalLocation.Region.StartColumn - if startLine != nil && startColumn != nil { - return strconv.Itoa(*startLine) + ":" + strconv.Itoa(*startColumn) - } - return "" -} - func ExtractRelativePath(resultPath string, projectRoot string) string { // Remove OS-specific file prefix resultPath = strings.TrimPrefix(resultPath, "file:///private") @@ -291,8 +254,69 @@ func GetRunRules(run *sarif.Run) []*sarif.ReportingDescriptor { } func GetInvocationWorkingDirectory(invocation *sarif.Invocation) string { - if invocation.WorkingDirectory != nil && invocation.WorkingDirectory.URI != nil { + if invocation != nil && invocation.WorkingDirectory != nil && invocation.WorkingDirectory.URI != nil { return *invocation.WorkingDirectory.URI } return "" } + +func CreateRunWithDummyResults(results ...*sarif.Result) *sarif.Run { + run := sarif.NewRunWithInformationURI("", "") + for _, result := range results { + if result.RuleID != nil { + run.AddRule(*result.RuleID) + } + run.AddResult(result) + } + return run +} + +func CreateResultWithLocations(msg, ruleId, level string, locations ...*sarif.Location) *sarif.Result { + return &sarif.Result{ + Message: *sarif.NewTextMessage(msg), + Locations: locations, + Level: &level, + RuleID: &ruleId, + } +} + +func CreateLocation(fileName string, startLine, startCol, endLine, endCol int, snippet string) *sarif.Location { + return &sarif.Location{ + PhysicalLocation: &sarif.PhysicalLocation{ + ArtifactLocation: &sarif.ArtifactLocation{URI: &fileName}, + Region: &sarif.Region{ + StartLine: &startLine, + StartColumn: &startCol, + EndLine: &endLine, + EndColumn: &endCol, + Snippet: &sarif.ArtifactContent{Text: &snippet}}}, + } +} + +func CreateDummyPassingResult(ruleId string) *sarif.Result { + kind := "pass" + return &sarif.Result{ + Kind: &kind, + RuleID: &ruleId, + } +} + +func CreateResultWithOneLocation(fileName string, startLine, startCol, endLine, endCol int, snippet, ruleId, level string) *sarif.Result { + return CreateResultWithLocations("", ruleId, level, CreateLocation(fileName, startLine, startCol, endLine, endCol, snippet)) +} + +func CreateCodeFlow(threadFlows ...*sarif.ThreadFlow) *sarif.CodeFlow { + flow := sarif.NewCodeFlow() + for _, threadFlow := range threadFlows { + flow.AddThreadFlow(threadFlow) + } + return flow +} + +func CreateThreadFlow(locations ...*sarif.Location) *sarif.ThreadFlow { + stackStrace := sarif.NewThreadFlow() + for _, location := range locations { + stackStrace.AddLocation(sarif.NewThreadFlowLocation().WithLocation(location)) + } + return stackStrace +} diff --git a/xray/utils/sarifutils_test.go b/xray/utils/sarifutils_test.go index 4e0031268..20a92de41 100644 --- a/xray/utils/sarifutils_test.go +++ b/xray/utils/sarifutils_test.go @@ -1,43 +1,645 @@ package utils import ( - "github.com/jfrog/gofrog/datastructures" + "testing" + "github.com/owenrumney/go-sarif/v2/sarif" + "github.com/stretchr/testify/assert" ) -func getRunWithDummyResults(results ...*sarif.Result) *sarif.Run { - run := sarif.NewRunWithInformationURI("", "") - ids := datastructures.MakeSet[string]() - for _, result := range results { - if !ids.Exists(*result.RuleID) { - run.Tool.Driver.Rules = append(run.Tool.Driver.Rules, sarif.NewRule(*result.RuleID)) - ids.Add(*result.RuleID) - } +func TestAggregateMultipleRunsIntoSingle(t *testing.T) { + tests := []struct { + runs []*sarif.Run + expectedOutput *sarif.Run + }{ + { + runs: []*sarif.Run{}, + expectedOutput: CreateRunWithDummyResults(), + }, + { + runs: []*sarif.Run{ + CreateRunWithDummyResults( + CreateDummyPassingResult("rule1"), + CreateResultWithOneLocation("file", 1, 2, 3, 4, "snippet", "rule2", "level"), + ).WithInvocations([]*sarif.Invocation{ + sarif.NewInvocation().WithWorkingDirectory(sarif.NewSimpleArtifactLocation("wd")), + }), + CreateRunWithDummyResults(), + }, + expectedOutput: CreateRunWithDummyResults( + CreateDummyPassingResult("rule1"), + CreateResultWithOneLocation("file", 1, 2, 3, 4, "snippet", "rule2", "level"), + ).WithInvocations([]*sarif.Invocation{ + sarif.NewInvocation().WithWorkingDirectory(sarif.NewSimpleArtifactLocation("wd")), + }), + }, + { + runs: []*sarif.Run{ + CreateRunWithDummyResults( + CreateDummyPassingResult("rule1"), + CreateResultWithOneLocation("file", 1, 2, 3, 4, "snippet", "rule2", "level"), + CreateResultWithOneLocation("file", 1, 2, 3, 4, "snippet", "rule3", "level"), + ).WithInvocations([]*sarif.Invocation{ + sarif.NewInvocation().WithWorkingDirectory(sarif.NewSimpleArtifactLocation("wd")), + }), + CreateRunWithDummyResults( + CreateResultWithLocations("msg", "rule2", "level", + CreateLocation("file", 1, 2, 3, 4, "snippet"), + CreateLocation("file2", 1, 2, 3, 4, "other-snippet"), + ), + CreateResultWithOneLocation("file", 5, 6, 7, 8, "snippet2", "rule2", "level"), + ).WithInvocations([]*sarif.Invocation{ + sarif.NewInvocation().WithWorkingDirectory(sarif.NewSimpleArtifactLocation("wd2")), + }), + }, + expectedOutput: CreateRunWithDummyResults( + // First run results + CreateDummyPassingResult("rule1"), + CreateResultWithOneLocation("file", 1, 2, 3, 4, "snippet", "rule2", "level"), + CreateResultWithOneLocation("file", 1, 2, 3, 4, "snippet", "rule3", "level"), + // Second run results + CreateResultWithLocations("msg", "rule2", "level", + CreateLocation("file", 1, 2, 3, 4, "snippet"), + CreateLocation("file2", 1, 2, 3, 4, "other-snippet"), + ), + CreateResultWithOneLocation("file", 5, 6, 7, 8, "snippet2", "rule2", "level"), + ).WithInvocations([]*sarif.Invocation{ + sarif.NewInvocation().WithWorkingDirectory(sarif.NewSimpleArtifactLocation("wd")), + sarif.NewInvocation().WithWorkingDirectory(sarif.NewSimpleArtifactLocation("wd2")), + }), + }, + } + + for _, test := range tests { + run := CreateRunWithDummyResults() + AggregateMultipleRunsIntoSingle(test.runs, run) + assert.Equal(t, test.expectedOutput, run) } - return run.WithResults(results) } -func getDummyPassingResult(ruleId string) *sarif.Result { - kind := "pass" - return &sarif.Result{ - Kind: &kind, - RuleID: &ruleId, +func TestGetLocationRelatedCodeFlowsFromResult(t *testing.T) { + tests := []struct { + result *sarif.Result + location *sarif.Location + expectedOutput []*sarif.CodeFlow + }{ + { + result: CreateDummyPassingResult("rule"), + location: CreateLocation("file", 0, 0, 0, 0, "snippet"), + expectedOutput: nil, + }, + { + result: CreateResultWithOneLocation("file", 0, 0, 0, 0, "snippet", "rule", "level"), + location: CreateLocation("file", 0, 0, 0, 0, "snippet"), + expectedOutput: nil, + }, + { + result: CreateResultWithOneLocation("file", 0, 0, 0, 0, "snippet", "rule", "level").WithCodeFlows([]*sarif.CodeFlow{CreateCodeFlow(CreateThreadFlow(CreateLocation("file", 0, 0, 0, 0, "snippet")))}), + location: CreateLocation("file2", 0, 0, 0, 0, "snippet"), + expectedOutput: nil, + }, + { + result: CreateResultWithOneLocation("file", 0, 0, 0, 0, "snippet", "rule", "level").WithCodeFlows([]*sarif.CodeFlow{CreateCodeFlow(CreateThreadFlow(CreateLocation("file", 0, 0, 0, 0, "snippet")))}), + location: CreateLocation("file", 0, 0, 0, 0, "snippet"), + expectedOutput: []*sarif.CodeFlow{CreateCodeFlow(CreateThreadFlow(CreateLocation("file", 0, 0, 0, 0, "snippet")))}, + }, + { + result: CreateResultWithOneLocation("file", 0, 0, 0, 0, "snippet", "rule", "level").WithCodeFlows([]*sarif.CodeFlow{ + CreateCodeFlow(CreateThreadFlow( + CreateLocation("file4", 2, 0, 2, 0, "snippetB"), + CreateLocation("file2", 0, 2, 0, 2, "snippetA"), + CreateLocation("file", 0, 0, 0, 0, "snippet"), + )), + CreateCodeFlow(CreateThreadFlow( + CreateLocation("file", 0, 0, 0, 0, "snippet"), + CreateLocation("file2", 1, 0, 1, 0, "snippet"), + )), + CreateCodeFlow(CreateThreadFlow( + CreateLocation("fileC", 1, 1, 1, 1, "snippetC"), + CreateLocation("file", 0, 0, 0, 0, "snippet"), + )), + }), + location: CreateLocation("file", 0, 0, 0, 0, "snippet"), + expectedOutput: []*sarif.CodeFlow{ + CreateCodeFlow(CreateThreadFlow( + CreateLocation("file4", 2, 0, 2, 0, "snippetB"), + CreateLocation("file2", 0, 2, 0, 2, "snippetA"), + CreateLocation("file", 0, 0, 0, 0, "snippet"), + )), + CreateCodeFlow(CreateThreadFlow( + CreateLocation("fileC", 1, 1, 1, 1, "snippetC"), + CreateLocation("file", 0, 0, 0, 0, "snippet"), + )), + }, + }, + } + + for _, test := range tests { + assert.Equal(t, test.expectedOutput, GetLocationRelatedCodeFlowsFromResult(test.location, test.result)) } } -func getDummyResultWithOneLocation(fileName string, startLine, startCol int, snippet, ruleId string, level string) *sarif.Result { - return &sarif.Result{ - Locations: []*sarif.Location{ - { - PhysicalLocation: &sarif.PhysicalLocation{ - ArtifactLocation: &sarif.ArtifactLocation{URI: &fileName}, - Region: &sarif.Region{ - StartLine: &startLine, - StartColumn: &startCol, - Snippet: &sarif.ArtifactContent{Text: &snippet}}}, +func TestGetResultsLocationCount(t *testing.T) { + tests := []struct { + runs []*sarif.Run + expectedOutput int + }{ + { + runs: []*sarif.Run{}, + expectedOutput: 0, + }, + { + runs: []*sarif.Run{CreateRunWithDummyResults()}, + expectedOutput: 0, + }, + { + runs: []*sarif.Run{CreateRunWithDummyResults( + CreateDummyPassingResult("rule"), + CreateResultWithOneLocation("file", 0, 0, 0, 0, "snippet", "rule", "level"), + )}, + expectedOutput: 1, + }, + { + runs: []*sarif.Run{ + CreateRunWithDummyResults( + CreateDummyPassingResult("rule"), + CreateResultWithOneLocation("file", 0, 0, 0, 0, "snippet", "rule", "level"), + ), + CreateRunWithDummyResults( + CreateResultWithLocations( + "msg", + "rule", + "level", + CreateLocation("file", 0, 0, 0, 0, "snippet"), + CreateLocation("file", 0, 0, 0, 0, "snippet"), + CreateLocation("file", 0, 0, 0, 0, "snippet"), + ), + ), }, + expectedOutput: 4, + }, + } + + for _, test := range tests { + assert.Equal(t, test.expectedOutput, GetResultsLocationCount(test.runs...)) + } +} + +func TestGetResultMsgText(t *testing.T) { + tests := []struct { + result *sarif.Result + expectedOutput string + }{ + { + result: &sarif.Result{}, + expectedOutput: "", + }, + { + result: CreateResultWithLocations("msg", "rule", "level"), + expectedOutput: "msg", + }, + } + + for _, test := range tests { + assert.Equal(t, test.expectedOutput, GetResultMsgText(test.result)) + } +} + +func TestGetLocationSnippet(t *testing.T) { + tests := []struct { + location *sarif.Location + expectedOutput string + }{ + { + location: nil, + expectedOutput: "", + }, + { + location: CreateLocation("filename", 1, 2, 3, 4, "snippet"), + expectedOutput: "snippet", + }, + } + + for _, test := range tests { + assert.Equal(t, test.expectedOutput, GetLocationSnippet(test.location)) + } +} + +func TestSetLocationSnippet(t *testing.T) { + tests := []struct { + location *sarif.Location + expectedOutput string + }{ + { + location: nil, + expectedOutput: "", + }, + { + location: CreateLocation("filename", 1, 2, 3, 4, "snippet"), + expectedOutput: "changedSnippet", + }, + } + + for _, test := range tests { + SetLocationSnippet(test.location, test.expectedOutput) + assert.Equal(t, test.expectedOutput, GetLocationSnippet(test.location)) + } +} + +func TestGetLocationFileName(t *testing.T) { + tests := []struct { + location *sarif.Location + expectedOutput string + }{ + { + location: nil, + expectedOutput: "", + }, + { + location: CreateLocation("filename", 1, 2, 3, 4, "snippet"), + expectedOutput: "filename", + }, + } + + for _, test := range tests { + assert.Equal(t, test.expectedOutput, GetLocationFileName(test.location)) + } +} + +func TestGetRelativeLocationFileName(t *testing.T) { + tests := []struct { + location *sarif.Location + invocations []*sarif.Invocation + expectedOutput string + }{ + { + location: CreateLocation("file:///root/someDir/another/file", 1, 2, 3, 4, "snippet"), + invocations: []*sarif.Invocation{}, + expectedOutput: "root/someDir/another/file", + }, + { + location: CreateLocation("file:///root/someDir/another/file", 1, 2, 3, 4, "snippet"), + invocations: []*sarif.Invocation{{WorkingDirectory: sarif.NewSimpleArtifactLocation("/not/relevant")}}, + expectedOutput: "root/someDir/another/file", + }, + { + location: CreateLocation("file:///root/someDir/another/file", 1, 2, 3, 4, "snippet"), + invocations: []*sarif.Invocation{{WorkingDirectory: sarif.NewSimpleArtifactLocation("/root/someDir/")}}, + expectedOutput: "another/file", + }, + } + + for _, test := range tests { + assert.Equal(t, test.expectedOutput, GetRelativeLocationFileName(test.location, test.invocations)) + } +} + +func TestSetLocationFileName(t *testing.T) { + tests := []struct { + location *sarif.Location + expectedOutput string + }{ + { + location: nil, + expectedOutput: "", + }, + { + location: CreateLocation("filename", 1, 2, 3, 4, "snippet"), + expectedOutput: "changedFilename", + }, + } + + for _, test := range tests { + SetLocationFileName(test.location, test.expectedOutput) + assert.Equal(t, test.expectedOutput, GetLocationFileName(test.location)) + } +} + +func TestGetLocationRegion(t *testing.T) { + tests := []struct { + location *sarif.Location + expectedOutput *sarif.Region + }{ + { + location: nil, + expectedOutput: nil, + }, + { + location: &sarif.Location{PhysicalLocation: &sarif.PhysicalLocation{}}, + expectedOutput: nil, + }, + { + location: CreateLocation("filename", 1, 2, 3, 4, "snippet"), + expectedOutput: sarif.NewRegion().WithStartLine(1).WithStartColumn(2).WithEndLine(3).WithEndColumn(4). + WithSnippet(sarif.NewArtifactContent().WithText("snippet")), + }, + } + + for _, test := range tests { + assert.Equal(t, test.expectedOutput, getLocationRegion(test.location)) + } +} + +func TestGetLocationStartLine(t *testing.T) { + tests := []struct { + location *sarif.Location + expectedOutput int + }{ + { + location: nil, + expectedOutput: 0, + }, + { + location: CreateLocation("filename", 1, 2, 3, 4, "snippet"), + expectedOutput: 1, + }, + } + + for _, test := range tests { + assert.Equal(t, test.expectedOutput, GetLocationStartLine(test.location)) + } +} + +func TestGetLocationStartColumn(t *testing.T) { + tests := []struct { + location *sarif.Location + expectedOutput int + }{ + { + location: nil, + expectedOutput: 0, + }, + { + location: CreateLocation("filename", 1, 2, 3, 4, "snippet"), + expectedOutput: 2, + }, + } + + for _, test := range tests { + assert.Equal(t, test.expectedOutput, GetLocationStartColumn(test.location)) + } +} + +func TestGetLocationEndLine(t *testing.T) { + tests := []struct { + location *sarif.Location + expectedOutput int + }{ + { + location: nil, + expectedOutput: 0, + }, + { + location: CreateLocation("filename", 1, 2, 3, 4, "snippet"), + expectedOutput: 3, + }, + } + + for _, test := range tests { + assert.Equal(t, test.expectedOutput, GetLocationEndLine(test.location)) + } +} + +func TestGetLocationEndColumn(t *testing.T) { + tests := []struct { + location *sarif.Location + expectedOutput int + }{ + { + location: nil, + expectedOutput: 0, + }, + { + location: CreateLocation("filename", 1, 2, 3, 4, "snippet"), + expectedOutput: 4, + }, + } + + for _, test := range tests { + assert.Equal(t, test.expectedOutput, GetLocationEndColumn(test.location)) + } +} + +func TestExtractRelativePath(t *testing.T) { + tests := []struct { + fullPath string + projectPath string + expectedResult string + }{ + {fullPath: "file:///Users/user/Desktop/secrets_scanner/tests/req.nodejs/file.js", + projectPath: "Users/user/Desktop/secrets_scanner/", expectedResult: "tests/req.nodejs/file.js"}, + {fullPath: "file:///private/Users/user/Desktop/secrets_scanner/tests/req.nodejs/file.js", + projectPath: "Users/user/Desktop/secrets_scanner/", expectedResult: "tests/req.nodejs/file.js"}, + {fullPath: "invalidFullPath", + projectPath: "Users/user/Desktop/secrets_scanner/", expectedResult: "invalidFullPath"}, + {fullPath: "", + projectPath: "Users/user/Desktop/secrets_scanner/", expectedResult: ""}, + {fullPath: "file:///Users/user/Desktop/secrets_scanner/tests/req.nodejs/file.js", + projectPath: "invalidProjectPath", expectedResult: "Users/user/Desktop/secrets_scanner/tests/req.nodejs/file.js"}, + {fullPath: "file:///private/Users/user/Desktop/secrets_scanner/tests/req.nodejs/file.js", + projectPath: "invalidProjectPath", expectedResult: "Users/user/Desktop/secrets_scanner/tests/req.nodejs/file.js"}, + } + + for _, test := range tests { + assert.Equal(t, test.expectedResult, ExtractRelativePath(test.fullPath, test.projectPath)) + } +} + +func TestGetResultSeverity(t *testing.T) { + levelValueHigh := string(errorLevel) + levelValueMedium := string(warningLevel) + levelValueMedium2 := string(infoLevel) + levelValueLow := string(noteLevel) + levelValueUnknown := string(noneLevel) + + tests := []struct { + result *sarif.Result + expectedSeverity string + }{ + {result: &sarif.Result{}, + expectedSeverity: "Medium"}, + {result: &sarif.Result{Level: &levelValueHigh}, + expectedSeverity: "High"}, + {result: &sarif.Result{Level: &levelValueMedium}, + expectedSeverity: "Medium"}, + {result: &sarif.Result{Level: &levelValueMedium2}, + expectedSeverity: "Medium"}, + {result: &sarif.Result{Level: &levelValueLow}, + expectedSeverity: "Low"}, + {result: &sarif.Result{Level: &levelValueUnknown}, + expectedSeverity: "Unknown"}, + } + + for _, test := range tests { + assert.Equal(t, test.expectedSeverity, GetResultSeverity(test.result)) + } +} + +func TestConvertToSarifLevel(t *testing.T) { + tests := []struct { + severity string + expectedOutput string + }{ + { + severity: "Unknown", + expectedOutput: "none", + }, + { + severity: "Low", + expectedOutput: "note", + }, + { + severity: "Medium", + expectedOutput: "warning", + }, + { + severity: "High", + expectedOutput: "error", + }, + { + severity: "Critical", + expectedOutput: "error", + }, + } + + for _, test := range tests { + assert.Equal(t, test.expectedOutput, ConvertToSarifLevel(test.severity)) + } +} + +func TestIsApplicableResult(t *testing.T) { + tests := []struct { + name string + sarifResult *sarif.Result + expectedOutput bool + }{ + { + sarifResult: CreateDummyPassingResult("rule"), + expectedOutput: false, + }, + { + sarifResult: CreateResultWithOneLocation("file", 0, 0, 0, 0, "snippet1", "ruleId1", "level1"), + expectedOutput: true, + }, + } + + for _, test := range tests { + assert.Equal(t, test.expectedOutput, IsApplicableResult(test.sarifResult)) + } +} + +func TestGetRuleFullDescription(t *testing.T) { + tests := []struct { + rule *sarif.ReportingDescriptor + expectedOutput string + }{ + { + rule: sarif.NewRule("rule"), + expectedOutput: "", + }, + { + rule: sarif.NewRule("rule").WithFullDescription(nil), + expectedOutput: "", + }, + { + rule: sarif.NewRule("rule").WithFullDescription(sarif.NewMultiformatMessageString("description")), + expectedOutput: "description", }, - Level: &level, - RuleID: &ruleId, + } + + for _, test := range tests { + assert.Equal(t, test.expectedOutput, GetRuleFullDescription(test.rule)) + } +} + +func TestCveToApplicabilityRuleId(t *testing.T) { + assert.Equal(t, "applic_cve", CveToApplicabilityRuleId("cve")) +} + +func TestApplicabilityRuleIdToCve(t *testing.T) { + tests := []struct { + ruleId string + expectedOutput string + }{ + { + ruleId: "rule", + expectedOutput: "rule", + }, + { + ruleId: "applic_cve", + expectedOutput: "cve", + }, + } + + for _, test := range tests { + assert.Equal(t, test.expectedOutput, ApplicabilityRuleIdToCve(test.ruleId)) + } +} + +func TestGetRunRules(t *testing.T) { + tests := []struct { + run *sarif.Run + expectedOutput []*sarif.ReportingDescriptor + }{ + { + run: &sarif.Run{}, + expectedOutput: []*sarif.ReportingDescriptor{}, + }, + { + run: CreateRunWithDummyResults(), + expectedOutput: []*sarif.ReportingDescriptor{}, + }, + { + run: CreateRunWithDummyResults( + CreateDummyPassingResult("rule1"), + ), + expectedOutput: []*sarif.ReportingDescriptor{sarif.NewRule("rule1")}, + }, + { + run: CreateRunWithDummyResults( + CreateDummyPassingResult("rule1"), + CreateDummyPassingResult("rule1"), + CreateDummyPassingResult("rule2"), + CreateDummyPassingResult("rule3"), + CreateDummyPassingResult("rule2"), + ), + expectedOutput: []*sarif.ReportingDescriptor{sarif.NewRule("rule1"), sarif.NewRule("rule2"), sarif.NewRule("rule3")}, + }, + } + + for _, test := range tests { + assert.Equal(t, test.expectedOutput, GetRunRules(test.run)) + } +} + +func TestGetInvocationWorkingDirectory(t *testing.T) { + tests := []struct { + invocation *sarif.Invocation + expectedOutput string + }{ + { + invocation: nil, + expectedOutput: "", + }, + { + invocation: sarif.NewInvocation(), + expectedOutput: "", + }, + { + invocation: sarif.NewInvocation().WithWorkingDirectory(nil), + expectedOutput: "", + }, + { + invocation: sarif.NewInvocation().WithWorkingDirectory(sarif.NewArtifactLocation()), + expectedOutput: "", + }, + { + invocation: sarif.NewInvocation().WithWorkingDirectory(sarif.NewArtifactLocation().WithUri("file_to_wd")), + expectedOutput: "file_to_wd", + }, + } + + for _, test := range tests { + assert.Equal(t, test.expectedOutput, GetInvocationWorkingDirectory(test.invocation)) } } From 7a180ec2ec7a48327834a485ad03d9d2b2154b08 Mon Sep 17 00:00:00 2001 From: Eyal Delarea Date: Tue, 26 Sep 2023 18:02:38 +0300 Subject: [PATCH 04/16] Refactor Resultwriter (#929) --- xray/commands/audit/audit.go | 18 ++-- xray/commands/scan/buildscan.go | 18 +++- xray/commands/scan/scan.go | 20 ++-- xray/utils/resultstable.go | 12 +-- xray/utils/resultstable_test.go | 2 +- xray/utils/resultwriter.go | 161 +++++++++++++++++++++----------- 6 files changed, 149 insertions(+), 82 deletions(-) diff --git a/xray/commands/audit/audit.go b/xray/commands/audit/audit.go index 2303ef0a2..1347e5357 100644 --- a/xray/commands/audit/audit.go +++ b/xray/commands/audit/audit.go @@ -109,15 +109,15 @@ func (auditCmd *AuditCommand) Run() (err error) { // Print Scan results on all cases except if errors accrued on SCA scan and no security/license issues found. printScanResults := !(auditResults.ScaError != nil && xrayutils.IsEmptyScanResponse(auditResults.ExtendedScanResults.XrayResults)) if printScanResults { - err = xrayutils.PrintScanResults(auditResults.ExtendedScanResults, - nil, - auditCmd.OutputFormat(), - auditCmd.IncludeVulnerabilities, - auditCmd.IncludeLicenses, - auditResults.IsMultipleRootProject, - auditCmd.PrintExtendedTable, false, messages, - ) - if err != nil { + if err = xrayutils.NewResultsWriter(auditResults.ExtendedScanResults). + SetIsMultipleRootProject(auditResults.IsMultipleRootProject). + SetIncludeVulnerabilities(auditCmd.IncludeVulnerabilities). + SetIncludeLicenses(auditCmd.IncludeLicenses). + SetOutputFormat(auditCmd.OutputFormat()). + SetPrintExtendedTable(auditCmd.PrintExtendedTable). + SetExtraMessages(messages). + SetScanType(services.Dependency). + PrintScanResults(); err != nil { return } } diff --git a/xray/commands/scan/buildscan.go b/xray/commands/scan/buildscan.go index 7dd6bdcd0..ec9f9a37e 100644 --- a/xray/commands/scan/buildscan.go +++ b/xray/commands/scan/buildscan.go @@ -128,22 +128,30 @@ func (bsc *BuildScanCommand) runBuildScanAndPrintResults(xrayManager *xray.XrayS extendedScanResults := &xrutils.ExtendedScanResults{XrayResults: scanResponse} + resultsPrinter := xrutils.NewResultsWriter(extendedScanResults). + SetOutputFormat(bsc.outputFormat). + SetIncludeVulnerabilities(bsc.includeVulnerabilities). + SetIncludeLicenses(false). + SetIsMultipleRootProject(true). + SetPrintExtendedTable(bsc.printExtendedTable). + SetScanType(services.Binary). + SetExtraMessages(nil) + if bsc.outputFormat != xrutils.Table { // Print the violations and/or vulnerabilities as part of one JSON. - err = xrutils.PrintScanResults(extendedScanResults, nil, bsc.outputFormat, false, false, false, bsc.printExtendedTable, true, nil) + err = resultsPrinter.PrintScanResults() } else { // Print two different tables for violations and vulnerabilities (if needed) // If "No Xray Fail build policy...." error received, no need to print violations if !noFailBuildPolicy { - err = xrutils.PrintScanResults(extendedScanResults, nil, bsc.outputFormat, false, false, false, bsc.printExtendedTable, true, nil) - if err != nil { + if err = resultsPrinter.PrintScanResults(); err != nil { return false, err } } if bsc.includeVulnerabilities { - err = xrutils.PrintScanResults(extendedScanResults, nil, bsc.outputFormat, true, false, false, bsc.printExtendedTable, true, nil) - if err != nil { + resultsPrinter.SetIncludeVulnerabilities(true) + if err = resultsPrinter.PrintScanResults(); err != nil { return false, err } } diff --git a/xray/commands/scan/scan.go b/xray/commands/scan/scan.go index b77a3c069..cd910ea0a 100644 --- a/xray/commands/scan/scan.go +++ b/xray/commands/scan/scan.go @@ -242,14 +242,18 @@ func (scanCmd *ScanCommand) Run() (err error) { scanErrors = appendErrorSlice(scanErrors, fileProducerErrors) scanErrors = appendErrorSlice(scanErrors, indexedFileProducerErrors) extendedScanResults := &xrutils.ExtendedScanResults{XrayResults: flatResults} - err = xrutils.PrintScanResults(extendedScanResults, - scanErrors, - scanCmd.outputFormat, - scanCmd.includeVulnerabilities, - scanCmd.includeLicenses, - true, - scanCmd.printExtendedTable, true, nil, - ) + + if err = xrutils.NewResultsWriter(extendedScanResults). + SetOutputFormat(scanCmd.outputFormat). + SetIncludeVulnerabilities(scanCmd.includeVulnerabilities). + SetIncludeLicenses(scanCmd.includeLicenses). + SetPrintExtendedTable(scanCmd.printExtendedTable). + SetIsMultipleRootProject(true). + SetScanType(services.Binary). + PrintScanResults(); err != nil { + return + } + if err != nil { return err } diff --git a/xray/utils/resultstable.go b/xray/utils/resultstable.go index 27fc58ccd..0b308b9ff 100644 --- a/xray/utils/resultstable.go +++ b/xray/utils/resultstable.go @@ -37,13 +37,13 @@ const ( // In case one (or more) of the violations contains the field FailBuild set to true, CliError with exit code 3 will be returned. // Set printExtended to true to print fields with 'extended' tag. // If the scan argument is set to true, print the scan tables. -func PrintViolationsTable(violations []services.Violation, extendedResults *ExtendedScanResults, multipleRoots, printExtended, isBinaryScan bool) error { +func PrintViolationsTable(violations []services.Violation, extendedResults *ExtendedScanResults, multipleRoots, printExtended bool, scanType services.ScanType) error { securityViolationsRows, licenseViolationsRows, operationalRiskViolationsRows, err := prepareViolations(violations, extendedResults, multipleRoots, true, true) if err != nil { return err } // Print tables, if scan is true; print the scan tables. - if isBinaryScan { + if scanType == services.Binary { err = coreutils.PrintTable(formats.ConvertToVulnerabilityScanTableRow(securityViolationsRows), "Security Violations", "No security violations were found", printExtended) if err != nil { return err @@ -182,13 +182,13 @@ func prepareViolations(violations []services.Violation, extendedResults *Extende // In case multipleRoots is true, the field Component will show the root of each impact path, otherwise it will show the root's child. // Set printExtended to true to print fields with 'extended' tag. // If the scan argument is set to true, print the scan tables. -func PrintVulnerabilitiesTable(vulnerabilities []services.Vulnerability, extendedResults *ExtendedScanResults, multipleRoots, printExtended, isBinaryScan bool) error { +func PrintVulnerabilitiesTable(vulnerabilities []services.Vulnerability, extendedResults *ExtendedScanResults, multipleRoots, printExtended bool, scanType services.ScanType) error { vulnerabilitiesRows, err := prepareVulnerabilities(vulnerabilities, extendedResults, multipleRoots, true, true) if err != nil { return err } - if isBinaryScan { + if scanType == services.Binary { return coreutils.PrintTable(formats.ConvertToVulnerabilityScanTableRow(vulnerabilitiesRows), "Vulnerable Components", "✨ No vulnerable components were found ✨", printExtended) } var emptyTableMessage string @@ -266,12 +266,12 @@ func sortVulnerabilityOrViolationRows(rows []formats.VulnerabilityOrViolationRow // In case multipleRoots is true, the field Component will show the root of each impact path, otherwise it will show the root's child. // Set printExtended to true to print fields with 'extended' tag. // If the scan argument is set to true, print the scan tables. -func PrintLicensesTable(licenses []services.License, printExtended, isBinaryScan bool) error { +func PrintLicensesTable(licenses []services.License, printExtended bool, scanType services.ScanType) error { licensesRows, err := PrepareLicenses(licenses) if err != nil { return err } - if isBinaryScan { + if scanType == services.Binary { return coreutils.PrintTable(formats.ConvertToLicenseScanTableRow(licensesRows), "Licenses", "No licenses were found", printExtended) } return coreutils.PrintTable(formats.ConvertToLicenseTableRow(licensesRows), "Licenses", "No licenses were found", printExtended) diff --git a/xray/utils/resultstable_test.go b/xray/utils/resultstable_test.go index 3606c1b8a..d0589fe24 100644 --- a/xray/utils/resultstable_test.go +++ b/xray/utils/resultstable_test.go @@ -25,7 +25,7 @@ func TestPrintViolationsTable(t *testing.T) { } for _, test := range tests { - err := PrintViolationsTable(test.violations, &ExtendedScanResults{}, false, true, true) + err := PrintViolationsTable(test.violations, &ExtendedScanResults{}, false, true, services.Binary) assert.NoError(t, err) if CheckIfFailBuild([]services.ScanResponse{{Violations: test.violations}}) { err = NewFailBuildError() diff --git a/xray/utils/resultwriter.go b/xray/utils/resultwriter.go index 1e8abcca4..c4fd62fdc 100644 --- a/xray/utils/resultwriter.go +++ b/xray/utils/resultwriter.go @@ -34,32 +34,88 @@ var OutputFormats = []string{string(Table), string(Json), string(SimpleJson), st var CurationOutputFormats = []string{string(Table), string(Json)} +type ResultsWriter struct { + // The scan results. + results *ExtendedScanResults + // SimpleJsonError Errors to be added to output of the SimpleJson format. + simpleJsonError []formats.SimpleJsonError + // Format The output format. + format OutputFormat + // IncludeVulnerabilities If true, include all vulnerabilities as part of the output. Else, include violations only. + includeVulnerabilities bool + // IncludeLicenses If true, also include license violations as part of the output. + includeLicenses bool + // IsMultipleRoots multipleRoots is set to true, in case the given results array contains (or may contain) results of several projects (like in binary scan). + isMultipleRoots bool + // PrintExtended, If true, show extended results. + printExtended bool + // The scanType (binary,dependency) + scanType services.ScanType + // Messages - Option array of messages, to be displayed if the format is Table + messages []string +} + +func NewResultsWriter(extendedScanResults *ExtendedScanResults) *ResultsWriter { + return &ResultsWriter{results: extendedScanResults} +} + +func (rw *ResultsWriter) SetOutputFormat(format OutputFormat) *ResultsWriter { + rw.format = format + return rw +} + +func (rw *ResultsWriter) SetScanType(scanType services.ScanType) *ResultsWriter { + rw.scanType = scanType + return rw +} + +func (rw *ResultsWriter) SetSimpleJsonError(jsonErrors []formats.SimpleJsonError) *ResultsWriter { + rw.simpleJsonError = jsonErrors + return rw +} + +func (rw *ResultsWriter) SetIncludeVulnerabilities(includeVulnerabilities bool) *ResultsWriter { + rw.includeVulnerabilities = includeVulnerabilities + return rw +} + +func (rw *ResultsWriter) SetIncludeLicenses(licenses bool) *ResultsWriter { + rw.includeLicenses = licenses + return rw +} + +func (rw *ResultsWriter) SetIsMultipleRootProject(isMultipleRootProject bool) *ResultsWriter { + rw.isMultipleRoots = isMultipleRootProject + return rw +} + +func (rw *ResultsWriter) SetPrintExtendedTable(extendedTable bool) *ResultsWriter { + rw.printExtended = extendedTable + return rw +} + +func (rw *ResultsWriter) SetExtraMessages(messages []string) *ResultsWriter { + rw.messages = messages + return rw + +} + // PrintScanResults prints the scan results in the specified format. // Note that errors are printed only with SimpleJson format. -// -// results - The scan results. -// simpleJsonError - Errors to be added to output of the SimpleJson format. -// format - The output format. -// includeVulnerabilities - If trie, include all vulnerabilities as part of the output. Else, include violations only. -// includeLicenses - If true, also include license violations as part of the output. -// isMultipleRoots - multipleRoots is set to true, in case the given results array contains (or may contain) results of several projects (like in binary scan). -// printExtended -If true, show extended results. -// scan - If true, use an output layout suitable for `jf scan` or `jf docker scan` results. Otherwise, use a layout compatible for `jf audit` . -// messages - Option array of messages, to be displayed if the format is Table -func PrintScanResults(results *ExtendedScanResults, simpleJsonError []formats.SimpleJsonError, format OutputFormat, includeVulnerabilities, includeLicenses, isMultipleRoots, printExtended, isBinaryScan bool, messages []string) error { - switch format { +func (rw *ResultsWriter) PrintScanResults() error { + switch rw.format { case Table: - return printScanResultsTables(results, isBinaryScan, includeVulnerabilities, includeLicenses, isMultipleRoots, printExtended, messages) + return rw.printScanResultsTables() case SimpleJson: - jsonTable, err := convertScanToSimpleJson(results, simpleJsonError, isMultipleRoots, includeLicenses, false) + jsonTable, err := rw.convertScanToSimpleJson() if err != nil { return err } return PrintJson(jsonTable) case Json: - return PrintJson(results.getXrayScanResults()) + return PrintJson(rw.results.getXrayScanResults()) case Sarif: - sarifFile, err := GenerateSarifContentFromResults(results, isMultipleRoots, includeLicenses, false) + sarifFile, err := rw.generateSarifContentFromResults(false) if err != nil { return err } @@ -67,41 +123,40 @@ func PrintScanResults(results *ExtendedScanResults, simpleJsonError []formats.Si } return nil } - -func printScanResultsTables(results *ExtendedScanResults, isBinaryScan, includeVulnerabilities, includeLicenses, isMultipleRoots, printExtended bool, messages []string) (err error) { - printMessages(messages) - violations, vulnerabilities, licenses := SplitScanResults(results.getXrayScanResults()) - if len(results.getXrayScanResults()) > 0 { +func (rw *ResultsWriter) printScanResultsTables() (err error) { + printMessages(rw.messages) + violations, vulnerabilities, licenses := SplitScanResults(rw.results.getXrayScanResults()) + if len(rw.results.getXrayScanResults()) > 0 { var resultsPath string - if resultsPath, err = writeJsonResults(results); err != nil { + if resultsPath, err = writeJsonResults(rw.results); err != nil { return } printMessage(coreutils.PrintTitle("The full scan results are available here: ") + coreutils.PrintLink(resultsPath)) } log.Output() - if includeVulnerabilities { - err = PrintVulnerabilitiesTable(vulnerabilities, results, isMultipleRoots, printExtended, isBinaryScan) + if rw.includeVulnerabilities { + err = PrintVulnerabilitiesTable(vulnerabilities, rw.results, rw.isMultipleRoots, rw.printExtended, rw.scanType) } else { - err = PrintViolationsTable(violations, results, isMultipleRoots, printExtended, isBinaryScan) + err = PrintViolationsTable(violations, rw.results, rw.isMultipleRoots, rw.printExtended, rw.scanType) } if err != nil { return } - if includeLicenses { - if err = PrintLicensesTable(licenses, printExtended, isBinaryScan); err != nil { + if rw.includeLicenses { + if err = PrintLicensesTable(licenses, rw.printExtended, rw.scanType); err != nil { return } } - if err = PrintSecretsTable(results.SecretsScanResults, results.EntitledForJas); err != nil { + if err = PrintSecretsTable(rw.results.SecretsScanResults, rw.results.EntitledForJas); err != nil { return } - if err = PrintIacTable(results.IacScanResults, results.EntitledForJas); err != nil { + if err = PrintIacTable(rw.results.IacScanResults, rw.results.EntitledForJas); err != nil { return } if !IsSastSupported() { return } - return PrintSastTable(results.SastScanResults, results.EntitledForJas) + return PrintSastTable(rw.results.SastScanResults, rw.results.EntitledForJas) } func printMessages(messages []string) { @@ -117,21 +172,21 @@ func printMessage(message string) { log.Output("💬" + message) } -func GenerateSarifContentFromResults(extendedResults *ExtendedScanResults, isMultipleRoots, includeLicenses, markdownOutput bool) (sarifStr string, err error) { +func (rw *ResultsWriter) generateSarifContentFromResults(markdownOutput bool) (sarifStr string, err error) { report, err := NewReport() if err != nil { return } - xrayRun, err := convertXrayResponsesToSarifRun(extendedResults, isMultipleRoots, includeLicenses, markdownOutput) + xrayRun, err := rw.convertXrayResponsesToSarifRun(markdownOutput) if err != nil { return } report.Runs = append(report.Runs, xrayRun) - report.Runs = append(report.Runs, extendedResults.ApplicabilityScanResults...) - report.Runs = append(report.Runs, extendedResults.IacScanResults...) - report.Runs = append(report.Runs, extendedResults.SecretsScanResults...) - report.Runs = append(report.Runs, extendedResults.SastScanResults...) + report.Runs = append(report.Runs, rw.results.ApplicabilityScanResults...) + report.Runs = append(report.Runs, rw.results.IacScanResults...) + report.Runs = append(report.Runs, rw.results.SecretsScanResults...) + report.Runs = append(report.Runs, rw.results.SastScanResults...) out, err := json.Marshal(report) if err != nil { @@ -141,13 +196,13 @@ func GenerateSarifContentFromResults(extendedResults *ExtendedScanResults, isMul return clientUtils.IndentJson(out), nil } -func convertXrayResponsesToSarifRun(extendedResults *ExtendedScanResults, isMultipleRoots, includeLicenses, markdownOutput bool) (run *sarif.Run, err error) { - xrayJson, err := convertXrayScanToSimpleJson(extendedResults, isMultipleRoots, includeLicenses, true) +func (rw *ResultsWriter) convertXrayResponsesToSarifRun(markdownOutput bool) (run *sarif.Run, err error) { + xrayJson, err := rw.convertXrayScanToSimpleJson(true) if err != nil { return } xrayRun := sarif.NewRunWithInformationURI("JFrog Xray Sca", "https://jfrog.com/xray/") - xrayRun.Tool.Driver.Version = &extendedResults.XrayVersion + xrayRun.Tool.Driver.Version = &rw.results.XrayVersion if len(xrayJson.Vulnerabilities) > 0 || len(xrayJson.SecurityViolations) > 0 { if err = extractXrayIssuesToSarifRun(xrayRun, xrayJson, markdownOutput); err != nil { return @@ -243,18 +298,18 @@ func addResultToSarifRun(issueId, msg, severity string, location *sarif.Location return } -func convertXrayScanToSimpleJson(extendedResults *ExtendedScanResults, isMultipleRoots, includeLicenses, simplifiedOutput bool) (formats.SimpleJsonResults, error) { - violations, vulnerabilities, licenses := SplitScanResults(extendedResults.XrayResults) +func (rw *ResultsWriter) convertXrayScanToSimpleJson(simplifiedOutput bool) (formats.SimpleJsonResults, error) { + violations, vulnerabilities, licenses := SplitScanResults(rw.results.XrayResults) jsonTable := formats.SimpleJsonResults{} if len(vulnerabilities) > 0 { - vulJsonTable, err := PrepareVulnerabilities(vulnerabilities, extendedResults, isMultipleRoots, simplifiedOutput) + vulJsonTable, err := PrepareVulnerabilities(vulnerabilities, rw.results, rw.isMultipleRoots, simplifiedOutput) if err != nil { return formats.SimpleJsonResults{}, err } jsonTable.Vulnerabilities = vulJsonTable } if len(violations) > 0 { - secViolationsJsonTable, licViolationsJsonTable, opRiskViolationsJsonTable, err := PrepareViolations(violations, extendedResults, isMultipleRoots, simplifiedOutput) + secViolationsJsonTable, licViolationsJsonTable, opRiskViolationsJsonTable, err := PrepareViolations(violations, rw.results, rw.isMultipleRoots, simplifiedOutput) if err != nil { return formats.SimpleJsonResults{}, err } @@ -262,7 +317,7 @@ func convertXrayScanToSimpleJson(extendedResults *ExtendedScanResults, isMultipl jsonTable.LicensesViolations = licViolationsJsonTable jsonTable.OperationalRiskViolations = opRiskViolationsJsonTable } - if includeLicenses { + if rw.includeLicenses { licJsonTable, err := PrepareLicenses(licenses) if err != nil { return formats.SimpleJsonResults{}, err @@ -273,21 +328,21 @@ func convertXrayScanToSimpleJson(extendedResults *ExtendedScanResults, isMultipl return jsonTable, nil } -func convertScanToSimpleJson(extendedResults *ExtendedScanResults, errors []formats.SimpleJsonError, isMultipleRoots, includeLicenses, simplifiedOutput bool) (formats.SimpleJsonResults, error) { - jsonTable, err := convertXrayScanToSimpleJson(extendedResults, isMultipleRoots, includeLicenses, simplifiedOutput) +func (rw *ResultsWriter) convertScanToSimpleJson() (formats.SimpleJsonResults, error) { + jsonTable, err := rw.convertXrayScanToSimpleJson(false) if err != nil { return formats.SimpleJsonResults{}, err } - if len(extendedResults.SecretsScanResults) > 0 { - jsonTable.Secrets = PrepareSecrets(extendedResults.SecretsScanResults) + if len(rw.results.SecretsScanResults) > 0 { + jsonTable.Secrets = PrepareSecrets(rw.results.SecretsScanResults) } - if len(extendedResults.IacScanResults) > 0 { - jsonTable.Iacs = PrepareIacs(extendedResults.IacScanResults) + if len(rw.results.IacScanResults) > 0 { + jsonTable.Iacs = PrepareIacs(rw.results.IacScanResults) } - if len(extendedResults.SastScanResults) > 0 { - jsonTable.Sast = PrepareSast(extendedResults.SastScanResults) + if len(rw.results.SastScanResults) > 0 { + jsonTable.Sast = PrepareSast(rw.results.SastScanResults) } - jsonTable.Errors = errors + jsonTable.Errors = rw.simpleJsonError return jsonTable, nil } From e2aca0fa17f80747f043643ea047fb41a270f411 Mon Sep 17 00:00:00 2001 From: Yahav Itzhak Date: Thu, 28 Sep 2023 14:30:28 +0300 Subject: [PATCH 05/16] Support JFrog Apps Config file (#943) --- go.mod | 1 + go.sum | 4 +- .../jas/applicability/applicabilitymanager.go | 30 ++-- .../applicabilitymanager_test.go | 18 +-- xray/commands/audit/jas/common.go | 74 +++++++++- xray/commands/audit/jas/commons_test.go | 129 ++++++++++++++++++ xray/commands/audit/jas/iac/iacscanner.go | 26 ++-- .../commands/audit/jas/iac/iacscanner_test.go | 12 +- xray/commands/audit/jas/sast/sastscanner.go | 53 ++++++- .../audit/jas/sast/sastscanner_test.go | 6 +- .../audit/jas/secrets/secretsscanner.go | 27 ++-- .../audit/jas/secrets/secretsscanner_test.go | 7 +- .../jas/testdata/.jfrog/jfrog-apps-config.yml | 50 +++++++ xray/utils/analyzermanager.go | 13 +- 14 files changed, 388 insertions(+), 62 deletions(-) create mode 100644 xray/commands/audit/jas/commons_test.go create mode 100644 xray/commands/audit/jas/testdata/.jfrog/jfrog-apps-config.yml diff --git a/go.mod b/go.mod index f4b687efc..e306e4516 100644 --- a/go.mod +++ b/go.mod @@ -14,6 +14,7 @@ require ( github.com/jedib0t/go-pretty/v6 v6.4.7 github.com/jfrog/build-info-go v1.9.10 github.com/jfrog/gofrog v1.3.0 + github.com/jfrog/jfrog-apps-config v1.0.1 github.com/jfrog/jfrog-client-go v1.32.3 github.com/magiconair/properties v1.8.7 github.com/manifoldco/promptui v0.9.0 diff --git a/go.sum b/go.sum index 737b839c2..8733c3d81 100644 --- a/go.sum +++ b/go.sum @@ -199,6 +199,8 @@ github.com/jfrog/build-info-go v1.9.10 h1:uXnDLVxpqxoAMpXcki00QaBB+M2BoGMMpHODPk github.com/jfrog/build-info-go v1.9.10/go.mod h1:ujJ8XQZMdT2tMkLSMJNyDd1pCY+duwHdjV+9or9FLIg= github.com/jfrog/gofrog v1.3.0 h1:o4zgsBZE4QyDbz2M7D4K6fXPTBJht+8lE87mS9bw7Gk= github.com/jfrog/gofrog v1.3.0/go.mod h1:IFMc+V/yf7rA5WZ74CSbXe+Lgf0iApEQLxRZVzKRUR0= +github.com/jfrog/jfrog-apps-config v1.0.1 h1:mtv6k7g8A8BVhlHGlSveapqf4mJfonwvXYLipdsOFMY= +github.com/jfrog/jfrog-apps-config v1.0.1/go.mod h1:8AIIr1oY9JuH5dylz2S6f8Ym2MaadPLR6noCBO4C22w= github.com/jfrog/jfrog-client-go v1.32.3 h1:B2M8Gu8EMrokbHWPPDgN1b7YRWwf0oe746epvQASK6c= github.com/jfrog/jfrog-client-go v1.32.3/go.mod h1:UewnwkIf/77HzBgwCPzOHZCK6V/Nw5/JwdzN/tRb4aU= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= @@ -319,7 +321,7 @@ github.com/vmihailenco/msgpack/v4 v4.3.12/go.mod h1:gborTTJjAo/GWTqqRjrLCn9pgNN+ github.com/vmihailenco/tagparser v0.1.1/go.mod h1:OeAg3pn3UbLjkWt+rN9oFYB6u/cQgqMEUPoW2WPyhdI= github.com/xanzy/ssh-agent v0.3.3 h1:+/15pJfg/RsTxqYcX6fHqOXZwwMP+2VyYWJeWM2qQFM= github.com/xanzy/ssh-agent v0.3.3/go.mod h1:6dzNDKs0J9rVPHPhaGCukekBHKqfl+L3KghI1Bc68Uw= -github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f h1:J9EGpcZtP0E/raorCMxlFGSTBrsSlaDGf3jU/qvAE2c= +github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb h1:zGWFAtiMcyryUHoUjUJX0/lt1H2+i2Ka2n+D3DImSNo= github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 h1:EzJWgHovont7NscjpAxXsDA8S8BMYve8Y5+7cuRE7R0= github.com/xeipuuv/gojsonschema v1.2.0 h1:LhYJRs+L4fBtjZUfuSZIKGeVu0QRy8e5Xi7D17UxZ74= github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8 h1:nIPpBwaJSVYIxUFsDv3M8ofmx9yWTog9BfvIu0q41lo= diff --git a/xray/commands/audit/jas/applicability/applicabilitymanager.go b/xray/commands/audit/jas/applicability/applicabilitymanager.go index b038e4371..c34318c00 100644 --- a/xray/commands/audit/jas/applicability/applicabilitymanager.go +++ b/xray/commands/audit/jas/applicability/applicabilitymanager.go @@ -3,6 +3,7 @@ package applicability import ( "path/filepath" + jfrogappsconfig "github.com/jfrog/jfrog-apps-config/go" "github.com/jfrog/jfrog-cli-core/v2/xray/commands/audit/jas" "github.com/jfrog/gofrog/datastructures" @@ -100,19 +101,22 @@ func isDirectComponents(components []string, directDependencies []string) bool { return false } -func (asm *ApplicabilityScanManager) Run(wd string) (err error) { - if len(asm.scanner.WorkingDirs) > 1 { - log.Info("Running applicability scanning in the", wd, "directory...") +func (asm *ApplicabilityScanManager) Run(module jfrogappsconfig.Module) (err error) { + if jas.ShouldSkipScanner(module, utils.Applicability) { + return + } + if len(asm.scanner.JFrogAppsConfig.Modules) > 1 { + log.Info("Running applicability scanning in the", module.SourceRoot, "directory...") } else { log.Info("Running applicability scanning...") } - if err = asm.createConfigFile(wd); err != nil { + if err = asm.createConfigFile(module); err != nil { return } if err = asm.runAnalyzerManager(); err != nil { return } - workingDirResults, err := jas.ReadJasScanRunsFromFile(asm.scanner.ResultsFileName, wd) + workingDirResults, err := jas.ReadJasScanRunsFromFile(asm.scanner.ResultsFileName, module.SourceRoot) if err != nil { return } @@ -141,25 +145,29 @@ type scanConfiguration struct { SkippedDirs []string `yaml:"skipped-folders"` } -func (asm *ApplicabilityScanManager) createConfigFile(workingDir string) error { - skipDirs := jas.SkippedDirs +func (asm *ApplicabilityScanManager) createConfigFile(module jfrogappsconfig.Module) error { + roots, err := jas.GetSourceRoots(module, nil) + if err != nil { + return err + } + excludePatterns := jas.GetExcludePatterns(module, nil) if asm.thirdPartyScan { log.Info("Including node modules folder in applicability scan") - skipDirs = removeElementFromSlice(skipDirs, jas.NodeModulesPattern) + excludePatterns = removeElementFromSlice(excludePatterns, jas.NodeModulesPattern) } configFileContent := applicabilityScanConfig{ Scans: []scanConfiguration{ { - Roots: []string{workingDir}, + Roots: roots, Output: asm.scanner.ResultsFileName, Type: applicabilityScanType, GrepDisable: false, CveWhitelist: asm.directDependenciesCves, - SkippedDirs: skipDirs, + SkippedDirs: excludePatterns, }, }, } - return jas.CreateScannersConfigFile(asm.scanner.ConfigFileName, configFileContent) + return jas.CreateScannersConfigFile(asm.scanner.ConfigFileName, configFileContent, utils.Applicability) } // Runs the analyzerManager app and returns a boolean to indicate whether the user is entitled for diff --git a/xray/commands/audit/jas/applicability/applicabilitymanager_test.go b/xray/commands/audit/jas/applicability/applicabilitymanager_test.go index b05d69130..98d54d28b 100644 --- a/xray/commands/audit/jas/applicability/applicabilitymanager_test.go +++ b/xray/commands/audit/jas/applicability/applicabilitymanager_test.go @@ -1,13 +1,15 @@ package applicability import ( + "os" + "path/filepath" + "testing" + + jfrogappsconfig "github.com/jfrog/jfrog-apps-config/go" "github.com/jfrog/jfrog-cli-core/v2/utils/coreutils" "github.com/jfrog/jfrog-cli-core/v2/xray/commands/audit/jas" "github.com/jfrog/jfrog-client-go/xray/services" "github.com/stretchr/testify/assert" - "os" - "path/filepath" - "testing" ) var mockDirectDependencies = []string{"issueId_2_direct_dependency", "issueId_1_direct_dependency"} @@ -36,7 +38,7 @@ func TestNewApplicabilityScanManager_DependencyTreeDoesntExist(t *testing.T) { // Assert if assert.NotNil(t, applicabilityManager) { assert.NotNil(t, applicabilityManager.scanner.ScannerDirCleanupFunc) - assert.Len(t, applicabilityManager.scanner.WorkingDirs, 1) + assert.Len(t, applicabilityManager.scanner.JFrogAppsConfig.Modules, 1) assert.NotEmpty(t, applicabilityManager.scanner.ConfigFileName) assert.NotEmpty(t, applicabilityManager.scanner.ResultsFileName) assert.Empty(t, applicabilityManager.directDependenciesCves) @@ -255,7 +257,7 @@ func TestCreateConfigFile_VerifyFileWasCreated(t *testing.T) { currWd, err := coreutils.GetWorkingDirectory() assert.NoError(t, err) - err = applicabilityManager.createConfigFile(currWd) + err = applicabilityManager.createConfigFile(jfrogappsconfig.Module{SourceRoot: currWd}) assert.NoError(t, err) defer func() { @@ -280,7 +282,7 @@ func TestParseResults_EmptyResults_AllCvesShouldGetUnknown(t *testing.T) { // Act var err error - applicabilityManager.applicabilityScanResults, err = jas.ReadJasScanRunsFromFile(applicabilityManager.scanner.ResultsFileName, scanner.WorkingDirs[0]) + applicabilityManager.applicabilityScanResults, err = jas.ReadJasScanRunsFromFile(applicabilityManager.scanner.ResultsFileName, scanner.JFrogAppsConfig.Modules[0].SourceRoot) if assert.NoError(t, err) { assert.Len(t, applicabilityManager.applicabilityScanResults, 1) @@ -297,7 +299,7 @@ func TestParseResults_ApplicableCveExist(t *testing.T) { // Act var err error - applicabilityManager.applicabilityScanResults, err = jas.ReadJasScanRunsFromFile(applicabilityManager.scanner.ResultsFileName, scanner.WorkingDirs[0]) + applicabilityManager.applicabilityScanResults, err = jas.ReadJasScanRunsFromFile(applicabilityManager.scanner.ResultsFileName, scanner.JFrogAppsConfig.Modules[0].SourceRoot) if assert.NoError(t, err) && assert.NotNil(t, applicabilityManager.applicabilityScanResults) { assert.Len(t, applicabilityManager.applicabilityScanResults, 1) @@ -314,7 +316,7 @@ func TestParseResults_AllCvesNotApplicable(t *testing.T) { // Act var err error - applicabilityManager.applicabilityScanResults, err = jas.ReadJasScanRunsFromFile(applicabilityManager.scanner.ResultsFileName, scanner.WorkingDirs[0]) + applicabilityManager.applicabilityScanResults, err = jas.ReadJasScanRunsFromFile(applicabilityManager.scanner.ResultsFileName, scanner.JFrogAppsConfig.Modules[0].SourceRoot) if assert.NoError(t, err) && assert.NotNil(t, applicabilityManager.applicabilityScanResults) { assert.Len(t, applicabilityManager.applicabilityScanResults, 1) diff --git a/xray/commands/audit/jas/common.go b/xray/commands/audit/jas/common.go index 2dfada8dc..fa7d508ef 100644 --- a/xray/commands/audit/jas/common.go +++ b/xray/commands/audit/jas/common.go @@ -2,20 +2,24 @@ package jas import ( "errors" + "fmt" "os" "path/filepath" "strings" "testing" + jfrogappsconfig "github.com/jfrog/jfrog-apps-config/go" rtutils "github.com/jfrog/jfrog-cli-core/v2/artifactory/utils" "github.com/jfrog/jfrog-cli-core/v2/utils/config" "github.com/jfrog/jfrog-cli-core/v2/utils/coreutils" "github.com/jfrog/jfrog-cli-core/v2/xray/utils" "github.com/jfrog/jfrog-client-go/utils/errorutils" "github.com/jfrog/jfrog-client-go/utils/io/fileutils" + "github.com/jfrog/jfrog-client-go/utils/log" "github.com/jfrog/jfrog-client-go/xray/services" "github.com/owenrumney/go-sarif/v2/sarif" "github.com/stretchr/testify/assert" + "golang.org/x/exp/slices" "gopkg.in/yaml.v3" ) @@ -24,7 +28,7 @@ const ( ) var ( - SkippedDirs = []string{"**/*test*/**", "**/*venv*/**", NodeModulesPattern, "**/*target*/**"} + DefaultExcludePatterns = []string{"**/*test*/**", "**/*venv*/**", NodeModulesPattern, "**/*target*/**"} mapSeverityToScore = map[string]string{ "": "0.0", @@ -41,7 +45,7 @@ type JasScanner struct { ResultsFileName string AnalyzerManager utils.AnalyzerManager ServerDetails *config.ServerDetails - WorkingDirs []string + JFrogAppsConfig *jfrogappsconfig.JFrogAppsConfig ScannerDirCleanupFunc func() error } @@ -60,22 +64,42 @@ func NewJasScanner(workingDirs []string, serverDetails *config.ServerDetails, mu scanner.ServerDetails = serverDetails scanner.ConfigFileName = filepath.Join(tempDir, "config.yaml") scanner.ResultsFileName = filepath.Join(tempDir, "results.sarif") - scanner.WorkingDirs, err = coreutils.GetFullPathsWorkingDirs(workingDirs) + scanner.JFrogAppsConfig, err = createJFrogAppsConfig(workingDirs) scanner.AnalyzerManager.MultiScanId = multiScanId return } +func createJFrogAppsConfig(workingDirs []string) (*jfrogappsconfig.JFrogAppsConfig, error) { + if jfrogAppsConfig, err := jfrogappsconfig.LoadConfigIfExist(); err != nil { + return nil, errorutils.CheckError(err) + } else if jfrogAppsConfig != nil { + // jfrog-apps-config.yml exist in the workspace + return jfrogAppsConfig, nil + } + + // jfrog-apps-config.yml does not exist in the workspace + fullPathsWorkingDirs, err := coreutils.GetFullPathsWorkingDirs(workingDirs) + if err != nil { + return nil, err + } + jfrogAppsConfig := new(jfrogappsconfig.JFrogAppsConfig) + for _, workingDir := range fullPathsWorkingDirs { + jfrogAppsConfig.Modules = append(jfrogAppsConfig.Modules, jfrogappsconfig.Module{SourceRoot: workingDir}) + } + return jfrogAppsConfig, nil +} + type ScannerCmd interface { - Run(wd string) (err error) + Run(module jfrogappsconfig.Module) (err error) } func (a *JasScanner) Run(scannerCmd ScannerCmd) (err error) { - for _, workingDir := range a.WorkingDirs { + for _, module := range a.JFrogAppsConfig.Modules { func() { defer func() { err = errors.Join(err, deleteJasProcessFiles(a.ConfigFileName, a.ResultsFileName)) }() - if err = scannerCmd.Run(workingDir); err != nil { + if err = scannerCmd.Run(module); err != nil { return } }() @@ -153,11 +177,12 @@ func convertToScore(severity string) string { return "" } -func CreateScannersConfigFile(fileName string, fileContent interface{}) error { +func CreateScannersConfigFile(fileName string, fileContent interface{}, scanType utils.JasScanType) error { yamlData, err := yaml.Marshal(&fileContent) if errorutils.CheckError(err) != nil { return err } + log.Debug(scanType.String() + " scanner input YAML:\n" + string(yamlData)) err = os.WriteFile(fileName, yamlData, 0644) return errorutils.CheckError(err) } @@ -196,3 +221,38 @@ func InitJasTest(t *testing.T, workingDirs ...string) (*JasScanner, func()) { func GetTestDataPath() string { return filepath.Join("..", "..", "..", "testdata") } + +func ShouldSkipScanner(module jfrogappsconfig.Module, scanType utils.JasScanType) bool { + lowerScanType := strings.ToLower(string(scanType)) + if slices.Contains(module.ExcludeScanners, lowerScanType) { + log.Info(fmt.Sprintf("Skipping %s scanning", scanType)) + return true + } + return false +} + +func GetSourceRoots(module jfrogappsconfig.Module, scanner *jfrogappsconfig.Scanner) ([]string, error) { + root, err := filepath.Abs(module.SourceRoot) + if err != nil { + return []string{}, errorutils.CheckError(err) + } + if scanner == nil || len(scanner.WorkingDirs) == 0 { + return []string{root}, errorutils.CheckError(err) + } + var roots []string + for _, workingDir := range scanner.WorkingDirs { + roots = append(roots, filepath.Join(root, workingDir)) + } + return roots, nil +} + +func GetExcludePatterns(module jfrogappsconfig.Module, scanner *jfrogappsconfig.Scanner) []string { + excludePatterns := module.ExcludePatterns + if scanner != nil { + excludePatterns = append(excludePatterns, scanner.ExcludePatterns...) + } + if len(excludePatterns) == 0 { + return DefaultExcludePatterns + } + return excludePatterns +} diff --git a/xray/commands/audit/jas/commons_test.go b/xray/commands/audit/jas/commons_test.go new file mode 100644 index 000000000..d600d7a73 --- /dev/null +++ b/xray/commands/audit/jas/commons_test.go @@ -0,0 +1,129 @@ +package jas + +import ( + "fmt" + "os" + "path/filepath" + "testing" + + jfrogappsconfig "github.com/jfrog/jfrog-apps-config/go" + clientTestUtils "github.com/jfrog/jfrog-client-go/utils/tests" + + "github.com/jfrog/jfrog-cli-core/v2/xray/utils" + "github.com/stretchr/testify/assert" +) + +var createJFrogAppsConfigCases = []struct { + workingDirs []string +}{ + {workingDirs: []string{}}, + {workingDirs: []string{"working-dir"}}, + {workingDirs: []string{"working-dir-1", "working-dir-2"}}, +} + +func TestCreateJFrogAppsConfig(t *testing.T) { + wd, err := os.Getwd() + assert.NoError(t, err) + + for _, testCase := range createJFrogAppsConfigCases { + t.Run(fmt.Sprintf("%v", testCase.workingDirs), func(t *testing.T) { + jfrogAppsConfig, err := createJFrogAppsConfig(testCase.workingDirs) + assert.NoError(t, err) + assert.NotNil(t, jfrogAppsConfig) + if len(testCase.workingDirs) == 0 { + assert.Len(t, jfrogAppsConfig.Modules, 1) + assert.Equal(t, wd, jfrogAppsConfig.Modules[0].SourceRoot) + return + } + assert.Len(t, jfrogAppsConfig.Modules, len(testCase.workingDirs)) + for i, workingDir := range testCase.workingDirs { + assert.Equal(t, filepath.Join(wd, workingDir), jfrogAppsConfig.Modules[i].SourceRoot) + } + }) + } +} + +func TestCreateJFrogAppsConfigWithConfig(t *testing.T) { + wd, err := os.Getwd() + assert.NoError(t, err) + chdirCallback := clientTestUtils.ChangeDirWithCallback(t, wd, "testdata") + defer chdirCallback() + + jfrogAppsConfig, err := createJFrogAppsConfig([]string{}) + assert.NoError(t, err) + assert.NotNil(t, jfrogAppsConfig) + assert.Equal(t, "1.0", jfrogAppsConfig.Version) + assert.Len(t, jfrogAppsConfig.Modules, 1) +} + +func TestShouldSkipScanner(t *testing.T) { + module := jfrogappsconfig.Module{} + assert.False(t, ShouldSkipScanner(module, utils.IaC)) + + module = jfrogappsconfig.Module{ExcludeScanners: []string{"sast"}} + assert.False(t, ShouldSkipScanner(module, utils.IaC)) + assert.True(t, ShouldSkipScanner(module, utils.Sast)) +} + +var getSourceRootsCases = []struct { + scanner *jfrogappsconfig.Scanner +}{ + {scanner: nil}, + {&jfrogappsconfig.Scanner{WorkingDirs: []string{"working-dir"}}}, + {&jfrogappsconfig.Scanner{WorkingDirs: []string{"working-dir-1", "working-dir-2"}}}, +} + +func TestGetSourceRoots(t *testing.T) { + testGetSourceRoots(t, "source-root") +} + +func TestGetSourceRootsEmptySourceRoot(t *testing.T) { + testGetSourceRoots(t, "") +} + +func testGetSourceRoots(t *testing.T, sourceRoot string) { + sourceRoot, err := filepath.Abs(sourceRoot) + assert.NoError(t, err) + module := jfrogappsconfig.Module{SourceRoot: sourceRoot} + for _, testCase := range getSourceRootsCases { + t.Run("", func(t *testing.T) { + scanner := testCase.scanner + actualSourceRoots, err := GetSourceRoots(module, scanner) + assert.NoError(t, err) + if scanner == nil { + assert.ElementsMatch(t, []string{module.SourceRoot}, actualSourceRoots) + return + } + expectedWorkingDirs := []string{} + for _, workingDir := range scanner.WorkingDirs { + expectedWorkingDirs = append(expectedWorkingDirs, filepath.Join(module.SourceRoot, workingDir)) + } + assert.ElementsMatch(t, actualSourceRoots, expectedWorkingDirs) + }) + } +} + +var getExcludePatternsCases = []struct { + scanner *jfrogappsconfig.Scanner +}{ + {scanner: nil}, + {&jfrogappsconfig.Scanner{WorkingDirs: []string{"exclude-dir"}}}, + {&jfrogappsconfig.Scanner{WorkingDirs: []string{"exclude-dir-1", "exclude-dir-2"}}}, +} + +func TestGetExcludePatterns(t *testing.T) { + module := jfrogappsconfig.Module{ExcludePatterns: []string{"exclude-root"}} + for _, testCase := range getExcludePatternsCases { + t.Run("", func(t *testing.T) { + scanner := testCase.scanner + actualExcludePatterns := GetExcludePatterns(module, scanner) + if scanner == nil { + assert.ElementsMatch(t, module.ExcludePatterns, actualExcludePatterns) + return + } + expectedExcludePatterns := module.ExcludePatterns + expectedExcludePatterns = append(expectedExcludePatterns, scanner.ExcludePatterns...) + assert.ElementsMatch(t, actualExcludePatterns, expectedExcludePatterns) + }) + } +} \ No newline at end of file diff --git a/xray/commands/audit/jas/iac/iacscanner.go b/xray/commands/audit/jas/iac/iacscanner.go index c4ccfdd39..f2ea4984b 100644 --- a/xray/commands/audit/jas/iac/iacscanner.go +++ b/xray/commands/audit/jas/iac/iacscanner.go @@ -1,9 +1,11 @@ package iac import ( - "github.com/jfrog/jfrog-cli-core/v2/xray/commands/audit/jas" "path/filepath" + jfrogappsconfig "github.com/jfrog/jfrog-apps-config/go" + "github.com/jfrog/jfrog-cli-core/v2/xray/commands/audit/jas" + "github.com/jfrog/jfrog-cli-core/v2/xray/utils" "github.com/jfrog/jfrog-client-go/utils/log" "github.com/owenrumney/go-sarif/v2/sarif" @@ -48,15 +50,17 @@ func newIacScanManager(scanner *jas.JasScanner) (manager *IacScanManager) { } } -func (iac *IacScanManager) Run(wd string) (err error) { - scanner := iac.scanner - if err = iac.createConfigFile(wd); err != nil { +func (iac *IacScanManager) Run(module jfrogappsconfig.Module) (err error) { + if jas.ShouldSkipScanner(module, utils.IaC) { + return + } + if err = iac.createConfigFile(module); err != nil { return } if err = iac.runAnalyzerManager(); err != nil { return } - workingDirResults, err := jas.ReadJasScanRunsFromFile(scanner.ResultsFileName, wd) + workingDirResults, err := jas.ReadJasScanRunsFromFile(iac.scanner.ResultsFileName, module.SourceRoot) if err != nil { return } @@ -75,18 +79,22 @@ type iacScanConfiguration struct { SkippedDirs []string `yaml:"skipped-folders"` } -func (iac *IacScanManager) createConfigFile(currentWd string) error { +func (iac *IacScanManager) createConfigFile(module jfrogappsconfig.Module) error { + roots, err := jas.GetSourceRoots(module, module.Scanners.Iac) + if err != nil { + return err + } configFileContent := iacScanConfig{ Scans: []iacScanConfiguration{ { - Roots: []string{currentWd}, + Roots: roots, Output: iac.scanner.ResultsFileName, Type: iacScannerType, - SkippedDirs: jas.SkippedDirs, + SkippedDirs: jas.GetExcludePatterns(module, module.Scanners.Iac), }, }, } - return jas.CreateScannersConfigFile(iac.scanner.ConfigFileName, configFileContent) + return jas.CreateScannersConfigFile(iac.scanner.ConfigFileName, configFileContent, utils.IaC) } func (iac *IacScanManager) runAnalyzerManager() error { diff --git a/xray/commands/audit/jas/iac/iacscanner_test.go b/xray/commands/audit/jas/iac/iacscanner_test.go index a2332421d..595715bfd 100644 --- a/xray/commands/audit/jas/iac/iacscanner_test.go +++ b/xray/commands/audit/jas/iac/iacscanner_test.go @@ -1,11 +1,13 @@ package iac import ( - "github.com/jfrog/jfrog-cli-core/v2/xray/commands/audit/jas" "os" "path/filepath" "testing" + jfrogappsconfig "github.com/jfrog/jfrog-apps-config/go" + "github.com/jfrog/jfrog-cli-core/v2/xray/commands/audit/jas" + "github.com/jfrog/jfrog-cli-core/v2/utils/coreutils" "github.com/stretchr/testify/assert" ) @@ -20,7 +22,7 @@ func TestNewIacScanManager(t *testing.T) { if assert.NotNil(t, iacScanManager) { assert.NotEmpty(t, iacScanManager.scanner.ConfigFileName) assert.NotEmpty(t, iacScanManager.scanner.ResultsFileName) - assert.NotEmpty(t, iacScanManager.scanner.WorkingDirs) + assert.NotEmpty(t, iacScanManager.scanner.JFrogAppsConfig.Modules[0].SourceRoot) assert.Equal(t, &jas.FakeServerDetails, iacScanManager.scanner.ServerDetails) } } @@ -33,7 +35,7 @@ func TestIacScan_CreateConfigFile_VerifyFileWasCreated(t *testing.T) { currWd, err := coreutils.GetWorkingDirectory() assert.NoError(t, err) - err = iacScanManager.createConfigFile(currWd) + err = iacScanManager.createConfigFile(jfrogappsconfig.Module{SourceRoot: currWd}) defer func() { err = os.Remove(iacScanManager.scanner.ConfigFileName) @@ -57,7 +59,7 @@ func TestIacParseResults_EmptyResults(t *testing.T) { // Act var err error - iacScanManager.iacScannerResults, err = jas.ReadJasScanRunsFromFile(iacScanManager.scanner.ResultsFileName, scanner.WorkingDirs[0]) + iacScanManager.iacScannerResults, err = jas.ReadJasScanRunsFromFile(iacScanManager.scanner.ResultsFileName, scanner.JFrogAppsConfig.Modules[0].SourceRoot) if assert.NoError(t, err) && assert.NotNil(t, iacScanManager.iacScannerResults) { assert.Len(t, iacScanManager.iacScannerResults, 1) assert.Empty(t, iacScanManager.iacScannerResults[0].Results) @@ -73,7 +75,7 @@ func TestIacParseResults_ResultsContainIacViolations(t *testing.T) { // Act var err error - iacScanManager.iacScannerResults, err = jas.ReadJasScanRunsFromFile(iacScanManager.scanner.ResultsFileName, scanner.WorkingDirs[0]) + iacScanManager.iacScannerResults, err = jas.ReadJasScanRunsFromFile(iacScanManager.scanner.ResultsFileName, scanner.JFrogAppsConfig.Modules[0].SourceRoot) if assert.NoError(t, err) && assert.NotNil(t, iacScanManager.iacScannerResults) { assert.Len(t, iacScanManager.iacScannerResults, 1) assert.Len(t, iacScanManager.iacScannerResults[0].Results, 4) diff --git a/xray/commands/audit/jas/sast/sastscanner.go b/xray/commands/audit/jas/sast/sastscanner.go index d4402cd68..2052a5361 100644 --- a/xray/commands/audit/jas/sast/sastscanner.go +++ b/xray/commands/audit/jas/sast/sastscanner.go @@ -3,6 +3,9 @@ package sast import ( "fmt" + "path/filepath" + + jfrogappsconfig "github.com/jfrog/jfrog-apps-config/go" "github.com/jfrog/jfrog-cli-core/v2/xray/commands/audit/jas" "github.com/jfrog/jfrog-cli-core/v2/xray/utils" "github.com/jfrog/jfrog-client-go/utils/log" @@ -11,6 +14,7 @@ import ( ) const ( + sastScannerType = "sast" sastScanCommand = "zd" ) @@ -40,12 +44,18 @@ func newSastScanManager(scanner *jas.JasScanner) (manager *SastScanManager) { } } -func (ssm *SastScanManager) Run(wd string) (err error) { +func (ssm *SastScanManager) Run(module jfrogappsconfig.Module) (err error) { + if jas.ShouldSkipScanner(module, utils.Sast) { + return + } + if err = ssm.createConfigFile(module); err != nil { + return + } scanner := ssm.scanner - if err = ssm.runAnalyzerManager(wd); err != nil { + if err = ssm.runAnalyzerManager(filepath.Dir(ssm.scanner.AnalyzerManager.AnalyzerManagerFullPath)); err != nil { return } - workingDirRuns, err := jas.ReadJasScanRunsFromFile(scanner.ResultsFileName, wd) + workingDirRuns, err := jas.ReadJasScanRunsFromFile(scanner.ResultsFileName, module.SourceRoot) if err != nil { return } @@ -54,8 +64,43 @@ func (ssm *SastScanManager) Run(wd string) (err error) { return } +type sastScanConfig struct { + Scans []scanConfiguration `yaml:"scans,omitempty"` +} + +type scanConfiguration struct { + Roots []string `yaml:"roots,omitempty"` + Type string `yaml:"type,omitempty"` + Language string `yaml:"language,omitempty"` + ExcludePatterns []string `yaml:"exclude_patterns,omitempty"` + ExcludedRules []string `yaml:"excluded-rules,omitempty"` +} + +func (ssm *SastScanManager) createConfigFile(module jfrogappsconfig.Module) error { + sastScanner := module.Scanners.Sast + if sastScanner == nil { + sastScanner = &jfrogappsconfig.SastScanner{} + } + roots, err := jas.GetSourceRoots(module, &sastScanner.Scanner) + if err != nil { + return err + } + configFileContent := sastScanConfig{ + Scans: []scanConfiguration{ + { + Type: sastScannerType, + Roots: roots, + Language: sastScanner.Language, + ExcludedRules: sastScanner.ExcludedRules, + ExcludePatterns: jas.GetExcludePatterns(module, &sastScanner.Scanner), + }, + }, + } + return jas.CreateScannersConfigFile(ssm.scanner.ConfigFileName, configFileContent, utils.Sast) +} + func (ssm *SastScanManager) runAnalyzerManager(wd string) error { - return ssm.scanner.AnalyzerManager.Exec(ssm.scanner.ResultsFileName, sastScanCommand, wd, ssm.scanner.ServerDetails) + return ssm.scanner.AnalyzerManager.ExecWithOutputFile(ssm.scanner.ConfigFileName, sastScanCommand, wd, ssm.scanner.ResultsFileName, ssm.scanner.ServerDetails) } // In the Sast scanner, there can be multiple results with the same location. diff --git a/xray/commands/audit/jas/sast/sastscanner_test.go b/xray/commands/audit/jas/sast/sastscanner_test.go index 6ed1980f3..036c028c8 100644 --- a/xray/commands/audit/jas/sast/sastscanner_test.go +++ b/xray/commands/audit/jas/sast/sastscanner_test.go @@ -21,7 +21,7 @@ func TestNewSastScanManager(t *testing.T) { if assert.NotNil(t, sastScanManager) { assert.NotEmpty(t, sastScanManager.scanner.ConfigFileName) assert.NotEmpty(t, sastScanManager.scanner.ResultsFileName) - assert.NotEmpty(t, sastScanManager.scanner.WorkingDirs) + assert.NotEmpty(t, sastScanManager.scanner.JFrogAppsConfig.Modules[0].SourceRoot) assert.Equal(t, &jas.FakeServerDetails, sastScanManager.scanner.ServerDetails) } } @@ -36,7 +36,7 @@ func TestSastParseResults_EmptyResults(t *testing.T) { // Act var err error - sastScanManager.sastScannerResults, err = jas.ReadJasScanRunsFromFile(sastScanManager.scanner.ResultsFileName, scanner.WorkingDirs[0]) + sastScanManager.sastScannerResults, err = jas.ReadJasScanRunsFromFile(sastScanManager.scanner.ResultsFileName, scanner.JFrogAppsConfig.Modules[0].SourceRoot) // Assert if assert.NoError(t, err) && assert.NotNil(t, sastScanManager.sastScannerResults) { @@ -57,7 +57,7 @@ func TestSastParseResults_ResultsContainIacViolations(t *testing.T) { // Act var err error - sastScanManager.sastScannerResults, err = jas.ReadJasScanRunsFromFile(sastScanManager.scanner.ResultsFileName, scanner.WorkingDirs[0]) + sastScanManager.sastScannerResults, err = jas.ReadJasScanRunsFromFile(sastScanManager.scanner.ResultsFileName, scanner.JFrogAppsConfig.Modules[0].SourceRoot) // Assert if assert.NoError(t, err) && assert.NotNil(t, sastScanManager.sastScannerResults) { diff --git a/xray/commands/audit/jas/secrets/secretsscanner.go b/xray/commands/audit/jas/secrets/secretsscanner.go index 6f0b985ff..eb383f6f4 100644 --- a/xray/commands/audit/jas/secrets/secretsscanner.go +++ b/xray/commands/audit/jas/secrets/secretsscanner.go @@ -4,6 +4,7 @@ import ( "path/filepath" "strings" + jfrogappsconfig "github.com/jfrog/jfrog-apps-config/go" "github.com/jfrog/jfrog-cli-core/v2/xray/commands/audit/jas" "github.com/jfrog/jfrog-cli-core/v2/xray/utils" "github.com/jfrog/jfrog-client-go/utils/log" @@ -48,19 +49,21 @@ func newSecretsScanManager(scanner *jas.JasScanner) (manager *SecretScanManager) } } -func (s *SecretScanManager) Run(wd string) (err error) { - scanner := s.scanner - if err = s.createConfigFile(wd); err != nil { +func (ssm *SecretScanManager) Run(module jfrogappsconfig.Module) (err error) { + if jas.ShouldSkipScanner(module, utils.Secrets) { return } - if err = s.runAnalyzerManager(); err != nil { + if err = ssm.createConfigFile(module); err != nil { return } - workingDirRuns, err := jas.ReadJasScanRunsFromFile(scanner.ResultsFileName, wd) + if err = ssm.runAnalyzerManager(); err != nil { + return + } + workingDirRuns, err := jas.ReadJasScanRunsFromFile(ssm.scanner.ResultsFileName, module.SourceRoot) if err != nil { return } - s.secretsScannerResults = append(s.secretsScannerResults, processSecretScanRuns(workingDirRuns)...) + ssm.secretsScannerResults = append(ssm.secretsScannerResults, processSecretScanRuns(workingDirRuns)...) return } @@ -75,18 +78,22 @@ type secretsScanConfiguration struct { SkippedDirs []string `yaml:"skipped-folders"` } -func (s *SecretScanManager) createConfigFile(currentWd string) error { +func (s *SecretScanManager) createConfigFile(module jfrogappsconfig.Module) error { + roots, err := jas.GetSourceRoots(module, module.Scanners.Secrets) + if err != nil { + return err + } configFileContent := secretsScanConfig{ Scans: []secretsScanConfiguration{ { - Roots: []string{currentWd}, + Roots: roots, Output: s.scanner.ResultsFileName, Type: secretsScannerType, - SkippedDirs: jas.SkippedDirs, + SkippedDirs: jas.GetExcludePatterns(module, module.Scanners.Secrets), }, }, } - return jas.CreateScannersConfigFile(s.scanner.ConfigFileName, configFileContent) + return jas.CreateScannersConfigFile(s.scanner.ConfigFileName, configFileContent, utils.Secrets) } func (s *SecretScanManager) runAnalyzerManager() error { diff --git a/xray/commands/audit/jas/secrets/secretsscanner_test.go b/xray/commands/audit/jas/secrets/secretsscanner_test.go index 14e917e16..bd9ec826d 100644 --- a/xray/commands/audit/jas/secrets/secretsscanner_test.go +++ b/xray/commands/audit/jas/secrets/secretsscanner_test.go @@ -5,6 +5,7 @@ import ( "path/filepath" "testing" + jfrogappsconfig "github.com/jfrog/jfrog-apps-config/go" "github.com/jfrog/jfrog-cli-core/v2/xray/commands/audit/jas" "github.com/jfrog/jfrog-cli-core/v2/utils/coreutils" @@ -29,7 +30,7 @@ func TestSecretsScan_CreateConfigFile_VerifyFileWasCreated(t *testing.T) { currWd, err := coreutils.GetWorkingDirectory() assert.NoError(t, err) - err = secretScanManager.createConfigFile(currWd) + err = secretScanManager.createConfigFile(jfrogappsconfig.Module{SourceRoot: currWd}) assert.NoError(t, err) defer func() { @@ -65,7 +66,7 @@ func TestParseResults_EmptyResults(t *testing.T) { // Act var err error - secretScanManager.secretsScannerResults, err = jas.ReadJasScanRunsFromFile(secretScanManager.scanner.ResultsFileName, scanner.WorkingDirs[0]) + secretScanManager.secretsScannerResults, err = jas.ReadJasScanRunsFromFile(secretScanManager.scanner.ResultsFileName, scanner.JFrogAppsConfig.Modules[0].SourceRoot) // Assert if assert.NoError(t, err) && assert.NotNil(t, secretScanManager.secretsScannerResults) { @@ -88,7 +89,7 @@ func TestParseResults_ResultsContainSecrets(t *testing.T) { // Act var err error - secretScanManager.secretsScannerResults, err = jas.ReadJasScanRunsFromFile(secretScanManager.scanner.ResultsFileName, scanner.WorkingDirs[0]) + secretScanManager.secretsScannerResults, err = jas.ReadJasScanRunsFromFile(secretScanManager.scanner.ResultsFileName, scanner.JFrogAppsConfig.Modules[0].SourceRoot) // Assert if assert.NoError(t, err) && assert.NotNil(t, secretScanManager.secretsScannerResults) { diff --git a/xray/commands/audit/jas/testdata/.jfrog/jfrog-apps-config.yml b/xray/commands/audit/jas/testdata/.jfrog/jfrog-apps-config.yml new file mode 100644 index 000000000..9357059d7 --- /dev/null +++ b/xray/commands/audit/jas/testdata/.jfrog/jfrog-apps-config.yml @@ -0,0 +1,50 @@ +# [Required] JFrog Applications Config version +version: "1.0" + +modules: + # [Required] Module name + - name: FrogLeapApp + # [Optional, default: "."] Application's root directory + source_root: "src" + # [Optional] Directories to exclude from scanning across all scanners + exclude_patterns: + - "docs/" + # [Optional] Scanners to exclude from JFrog Advanced Security (Options: "secrets", "sast", "iac") + exclude_scanners: + - secrets + # [Optional] Customize scanner configurations + scanners: + # [Optional] Configuration for Static Application Security Testing (SAST) + sast: + # [Optional] Specify the programming language for SAST + language: java + # [Optional] Working directories specific to SAST (Relative to source_root) + working_dirs: + - "src/module1" + - "src/module2" + # [Optional] Additional exclude patterns for this scanner + exclude_patterns: + - "src/module1/test" + # [Optional] List of specific scan rules to exclude from the scan + excluded_rules: + - xss-injection + + # [Optional] Configuration for secrets scan + secrets: + # [Optional] Working directories specific to the secret scanner (Relative to source_root) + working_dirs: + - "src/module1" + - "src/module2" + # [Optional] Additional exclude patterns for this scanner + exclude_patterns: + - "src/module1/test" + + # [Optional] Configuration for Infrastructure as Code scan (IaC) + iac: + # [Optional] Working directories specific to IaC (Relative to source_root) + working_dirs: + - "src/module1" + - "src/module2" + # [Optional] Additional exclude patterns for this Scanner + exclude_patterns: + - "src/module1/test" \ No newline at end of file diff --git a/xray/utils/analyzermanager.go b/xray/utils/analyzermanager.go index 12784921c..9866692de 100644 --- a/xray/utils/analyzermanager.go +++ b/xray/utils/analyzermanager.go @@ -102,10 +102,21 @@ type AnalyzerManager struct { } func (am *AnalyzerManager) Exec(configFile, scanCommand, workingDir string, serverDetails *config.ServerDetails) (err error) { + return am.ExecWithOutputFile(configFile, scanCommand, workingDir, "", serverDetails) +} + +func (am *AnalyzerManager) ExecWithOutputFile(configFile, scanCommand, workingDir, outputFile string, serverDetails *config.ServerDetails) (err error) { if err = SetAnalyzerManagerEnvVariables(serverDetails); err != nil { return } - cmd := exec.Command(am.AnalyzerManagerFullPath, scanCommand, configFile, am.MultiScanId) + var cmd *exec.Cmd + if len(outputFile) > 0 { + log.Debug("Executing", am.AnalyzerManagerFullPath, scanCommand, configFile, outputFile, am.MultiScanId) + cmd = exec.Command(am.AnalyzerManagerFullPath, scanCommand, configFile, outputFile, am.MultiScanId) + } else { + log.Debug("Executing", am.AnalyzerManagerFullPath, scanCommand, configFile, am.MultiScanId) + cmd = exec.Command(am.AnalyzerManagerFullPath, scanCommand, configFile, am.MultiScanId) + } defer func() { if cmd.ProcessState != nil && !cmd.ProcessState.Exited() { if killProcessError := cmd.Process.Kill(); errorutils.CheckError(killProcessError) != nil { From 12f17a82dea08fcf81b4ec7e816fe01ab4067934 Mon Sep 17 00:00:00 2001 From: Tal Arian Date: Thu, 28 Sep 2023 17:12:56 +0300 Subject: [PATCH 06/16] Report Usage - Improve Errors Messages (#979) --- utils/usage/usage.go | 40 +++++++++------------------------------ utils/usage/usage_test.go | 32 +------------------------------ 2 files changed, 10 insertions(+), 62 deletions(-) diff --git a/utils/usage/usage.go b/utils/usage/usage.go index 30f40f18c..9b493637b 100644 --- a/utils/usage/usage.go +++ b/utils/usage/usage.go @@ -2,7 +2,6 @@ package usage import ( "fmt" - "golang.org/x/sync/errgroup" "github.com/jfrog/jfrog-cli-core/v2/artifactory/utils" "github.com/jfrog/jfrog-cli-core/v2/utils/config" @@ -12,7 +11,7 @@ import ( "github.com/jfrog/jfrog-client-go/utils/errorutils" "github.com/jfrog/jfrog-client-go/utils/log" ecosysusage "github.com/jfrog/jfrog-client-go/utils/usage" - xrayusage "github.com/jfrog/jfrog-client-go/xray/usage" + "golang.org/x/sync/errgroup" ) const ( @@ -47,7 +46,6 @@ func NewUsageReporter(productId string, serverDetails *config.ServerDetails) *Us serverDetails: serverDetails, reportWaitGroup: new(errgroup.Group), sendToEcosystem: true, - sendToXray: true, sendToArtifactory: true, } } @@ -110,12 +108,15 @@ func (ur *UsageReporter) WaitForResponses() (err error) { func (ur *UsageReporter) reportToEcosystem(features ...ReportFeature) (err error) { if ur.serverDetails.Url == "" { - err = errorutils.CheckErrorf("platform Url is not set") + err = errorutils.CheckErrorf("platform URL is not set") return } reports, err := ur.convertAttributesToEcosystemReports(features...) - if len(reports) == 0 || err != nil { - err = errorutils.CheckErrorf("Nothing to send.") + if err != nil { + return + } + if len(reports) == 0 { + err = errorutils.CheckErrorf("nothing to send") return } return ecosysusage.SendEcosystemUsageReports(reports...) @@ -123,7 +124,7 @@ func (ur *UsageReporter) reportToEcosystem(features ...ReportFeature) (err error func (ur *UsageReporter) reportToArtifactory(features ...ReportFeature) (err error) { if ur.serverDetails.ArtifactoryUrl == "" { - err = errorutils.CheckErrorf("Artifactory Url is not set..") + err = errorutils.CheckErrorf("Artifactory URL is not set") return } serviceManager, err := utils.CreateServiceManager(ur.serverDetails, -1, 0, false) @@ -132,7 +133,7 @@ func (ur *UsageReporter) reportToArtifactory(features ...ReportFeature) (err err } converted := ur.convertAttributesToArtifactoryFeatures(features...) if len(converted) == 0 { - err = errorutils.CheckErrorf("Nothing to send.") + err = errorutils.CheckErrorf("nothing to send") return } return usage.ReportUsageToArtifactory(ur.ProductId, serviceManager, converted...) @@ -163,29 +164,6 @@ func (ur *UsageReporter) convertAttributesToArtifactoryFeatures(reportFeatures . return } -func (ur *UsageReporter) convertAttributesToXrayEvents(reportFeatures ...ReportFeature) (events []xrayusage.ReportXrayEventData) { - for _, feature := range reportFeatures { - convertedAttributes := []xrayusage.ReportUsageAttribute{} - for _, attribute := range feature.Attributes { - convertedAttributes = append(convertedAttributes, xrayusage.ReportUsageAttribute{ - AttributeName: attribute.AttributeName, - AttributeValue: attribute.AttributeValue, - }) - } - if feature.ClientId != "" { - // Add clientId as attribute - convertedAttributes = append(convertedAttributes, xrayusage.ReportUsageAttribute{ - AttributeName: clientIdAttributeName, - AttributeValue: feature.ClientId, - }) - } - events = append(events, xrayusage.CreateUsageEvent( - ur.ProductId, feature.FeatureId, convertedAttributes..., - )) - } - return -} - func (ur *UsageReporter) convertAttributesToEcosystemReports(reportFeatures ...ReportFeature) (reports []ecosysusage.ReportEcosystemUsageData, err error) { accountId := ur.serverDetails.Url clientToFeaturesMap := map[string][]string{} diff --git a/utils/usage/usage_test.go b/utils/usage/usage_test.go index 657f19ba1..c3b2b08a1 100644 --- a/utils/usage/usage_test.go +++ b/utils/usage/usage_test.go @@ -10,7 +10,6 @@ import ( "github.com/jfrog/jfrog-cli-core/v2/utils/config" "github.com/jfrog/jfrog-client-go/artifactory/usage" ecosysusage "github.com/jfrog/jfrog-client-go/utils/usage" - xrayusage "github.com/jfrog/jfrog-client-go/xray/usage" "github.com/stretchr/testify/assert" ) @@ -40,28 +39,6 @@ var ( {FeatureId: "featureId", ClientId: "clientId2"}, {FeatureId: "featureId"}, } - xrayEvents = []xrayusage.ReportXrayEventData{ - { - ProductId: productName, - EventId: "server_" + productName + "_featureId2", - Attributes: map[string]string{ - "clientId": "clientId", - "attribute": "value", - }, - Origin: "API_CLI", - }, - { - ProductId: productName, - EventId: "server_" + productName + "_featureId", - Attributes: map[string]string{"clientId": "clientId2"}, - Origin: "API_CLI", - }, - { - ProductId: productName, - EventId: "server_" + productName + "_featureId", - Origin: "API_CLI", - }, - } ecosystemData = []ecosysusage.ReportEcosystemUsageData{ { ProductId: productName, @@ -90,13 +67,6 @@ func TestConvertToArtifactoryUsage(t *testing.T) { } } -func TestConvertToXrayUsage(t *testing.T) { - reporter := NewUsageReporter(productName, &config.ServerDetails{XrayUrl: serverUrl + "/"}) - for i := 0; i < len(features); i++ { - assert.Equal(t, xrayEvents[i], reporter.convertAttributesToXrayEvents(features[i])[0]) - } -} - func TestConvertToEcosystemUsage(t *testing.T) { reporter := NewUsageReporter(productName, &config.ServerDetails{Url: serverUrl}) for i := 0; i < len(features); i++ { @@ -173,7 +143,7 @@ func TestReportEcosystemUsageError(t *testing.T) { } func create404UsageHandler(t *testing.T) http.HandlerFunc { - return func(w http.ResponseWriter, r *http.Request) { + return func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(http.StatusNotFound) } } From 1d74324157cb3f908499fe05c2d94a3a385a8208 Mon Sep 17 00:00:00 2001 From: Michael Sverdlov Date: Thu, 28 Sep 2023 17:56:40 +0300 Subject: [PATCH 07/16] Improve npm logs and error handling (#977) --- artifactory/commands/golang/go.go | 2 +- artifactory/commands/npm/npmcommand.go | 177 +++++++++--------- .../commands/transferconfig/transferconfig.go | 6 +- .../transferconfigmerge_test.go | 28 ++- artifactory/commands/utils/npmcmdutils.go | 12 +- artifactory/utils/dependenciesutils.go | 2 +- general/envsetup/envsetup.go | 4 +- general/invite/invite.go | 13 +- go.mod | 8 +- go.sum | 16 +- xray/commands/audit/scarunner.go | 2 +- 11 files changed, 133 insertions(+), 137 deletions(-) diff --git a/artifactory/commands/golang/go.go b/artifactory/commands/golang/go.go index 93ce330e0..68ab8002f 100644 --- a/artifactory/commands/golang/go.go +++ b/artifactory/commands/golang/go.go @@ -219,7 +219,7 @@ func copyGoPackageFiles(destPath, packageName, rtTargetRepo string, authArtDetai return fmt.Errorf("couldn't find suitable package files: %s", packageFilesPath) } // Set permission recursively - return coreutils.SetPermissionsRecursively(destPath, 0700) + return coreutils.SetPermissionsRecursively(destPath, 0755) } // getPackageFilePathFromArtifactory returns a string that represents the package files cache path. diff --git a/artifactory/commands/npm/npmcommand.go b/artifactory/commands/npm/npmcommand.go index c6b97cff7..7d6061a2e 100644 --- a/artifactory/commands/npm/npmcommand.go +++ b/artifactory/commands/npm/npmcommand.go @@ -63,108 +63,108 @@ func NewNpmCiCommand() *NpmCommand { return &NpmCommand{cmdName: "ci", internalCommandName: "rt_npm_ci"} } -func (ca *NpmCommand) CommandName() string { - return ca.internalCommandName +func (nc *NpmCommand) CommandName() string { + return nc.internalCommandName } -func (ca *NpmCommand) SetConfigFilePath(configFilePath string) *NpmCommand { - ca.configFilePath = configFilePath - return ca +func (nc *NpmCommand) SetConfigFilePath(configFilePath string) *NpmCommand { + nc.configFilePath = configFilePath + return nc } -func (ca *NpmCommand) SetArgs(args []string) *NpmCommand { - ca.npmArgs = args - return ca +func (nc *NpmCommand) SetArgs(args []string) *NpmCommand { + nc.npmArgs = args + return nc } -func (ca *NpmCommand) SetRepoConfig(conf *utils.RepositoryConfig) *NpmCommand { +func (nc *NpmCommand) SetRepoConfig(conf *utils.RepositoryConfig) *NpmCommand { serverDetails, _ := conf.ServerDetails() - ca.SetRepo(conf.TargetRepo()).SetServerDetails(serverDetails) - return ca + nc.SetRepo(conf.TargetRepo()).SetServerDetails(serverDetails) + return nc } -func (ca *NpmCommand) SetServerDetails(serverDetails *config.ServerDetails) *NpmCommand { - ca.serverDetails = serverDetails - return ca +func (nc *NpmCommand) SetServerDetails(serverDetails *config.ServerDetails) *NpmCommand { + nc.serverDetails = serverDetails + return nc } -func (ca *NpmCommand) SetRepo(repo string) *NpmCommand { - ca.repo = repo - return ca +func (nc *NpmCommand) SetRepo(repo string) *NpmCommand { + nc.repo = repo + return nc } -func (ca *NpmCommand) Init() error { +func (nc *NpmCommand) Init() error { // Read config file. - log.Debug("Preparing to read the config file", ca.configFilePath) - vConfig, err := utils.ReadConfigFile(ca.configFilePath, utils.YAML) + log.Debug("Preparing to read the config file", nc.configFilePath) + vConfig, err := utils.ReadConfigFile(nc.configFilePath, utils.YAML) if err != nil { return err } // Extract resolution params. - resolverParams, err := utils.GetRepoConfigByPrefix(ca.configFilePath, utils.ProjectConfigResolverPrefix, vConfig) + resolverParams, err := utils.GetRepoConfigByPrefix(nc.configFilePath, utils.ProjectConfigResolverPrefix, vConfig) if err != nil { return err } - _, _, _, filteredNpmArgs, buildConfiguration, err := commandUtils.ExtractNpmOptionsFromArgs(ca.npmArgs) + _, _, _, filteredNpmArgs, buildConfiguration, err := commandUtils.ExtractNpmOptionsFromArgs(nc.npmArgs) if err != nil { return err } - ca.SetRepoConfig(resolverParams).SetArgs(filteredNpmArgs).SetBuildConfiguration(buildConfiguration) + nc.SetRepoConfig(resolverParams).SetArgs(filteredNpmArgs).SetBuildConfiguration(buildConfiguration) return nil } -func (ca *NpmCommand) SetBuildConfiguration(buildConfiguration *utils.BuildConfiguration) *NpmCommand { - ca.buildConfiguration = buildConfiguration - return ca +func (nc *NpmCommand) SetBuildConfiguration(buildConfiguration *utils.BuildConfiguration) *NpmCommand { + nc.buildConfiguration = buildConfiguration + return nc } -func (ca *NpmCommand) ServerDetails() (*config.ServerDetails, error) { - return ca.serverDetails, nil +func (nc *NpmCommand) ServerDetails() (*config.ServerDetails, error) { + return nc.serverDetails, nil } -func (ca *NpmCommand) RestoreNpmrcFunc() func() error { - return ca.restoreNpmrcFunc +func (nc *NpmCommand) RestoreNpmrcFunc() func() error { + return nc.restoreNpmrcFunc } -func (ca *NpmCommand) PreparePrerequisites(repo string) error { +func (nc *NpmCommand) PreparePrerequisites(repo string) error { log.Debug("Preparing prerequisites...") var err error - ca.npmVersion, ca.executablePath, err = biutils.GetNpmVersionAndExecPath(log.Logger) + nc.npmVersion, nc.executablePath, err = biutils.GetNpmVersionAndExecPath(log.Logger) if err != nil { return err } - if ca.npmVersion.Compare(minSupportedNpmVersion) > 0 { + if nc.npmVersion.Compare(minSupportedNpmVersion) > 0 { return errorutils.CheckErrorf( - "JFrog CLI npm %s command requires npm client version %s or higher. The Current version is: %s", ca.cmdName, minSupportedNpmVersion, ca.npmVersion.GetVersion()) + "JFrog CLI npm %s command requires npm client version %s or higher. The Current version is: %s", nc.cmdName, minSupportedNpmVersion, nc.npmVersion.GetVersion()) } - if err := ca.setJsonOutput(); err != nil { + if err = nc.setJsonOutput(); err != nil { return err } - ca.workingDirectory, err = coreutils.GetWorkingDirectory() + nc.workingDirectory, err = coreutils.GetWorkingDirectory() if err != nil { return err } - log.Debug("Working directory set to:", ca.workingDirectory) - if err = ca.setArtifactoryAuth(); err != nil { + log.Debug("Working directory set to:", nc.workingDirectory) + if err = nc.setArtifactoryAuth(); err != nil { return err } - ca.npmAuth, ca.registry, err = commandUtils.GetArtifactoryNpmRepoDetails(repo, &ca.authArtDetails) + nc.npmAuth, nc.registry, err = commandUtils.GetArtifactoryNpmRepoDetails(repo, &nc.authArtDetails) if err != nil { return err } - return ca.setRestoreNpmrcFunc() + return nc.setRestoreNpmrcFunc() } -func (ca *NpmCommand) setRestoreNpmrcFunc() error { - restoreNpmrcFunc, err := commandUtils.BackupFile(filepath.Join(ca.workingDirectory, npmrcFileName), npmrcBackupFileName) +func (nc *NpmCommand) setRestoreNpmrcFunc() error { + restoreNpmrcFunc, err := commandUtils.BackupFile(filepath.Join(nc.workingDirectory, npmrcFileName), npmrcBackupFileName) if err != nil { return err } - ca.restoreNpmrcFunc = func() error { + nc.restoreNpmrcFunc = func() error { if unsetEnvErr := os.Unsetenv(npmConfigAuthEnv); unsetEnvErr != nil { return unsetEnvErr } @@ -173,43 +173,43 @@ func (ca *NpmCommand) setRestoreNpmrcFunc() error { return nil } -func (ca *NpmCommand) setArtifactoryAuth() error { - authArtDetails, err := ca.serverDetails.CreateArtAuthConfig() +func (nc *NpmCommand) setArtifactoryAuth() error { + authArtDetails, err := nc.serverDetails.CreateArtAuthConfig() if err != nil { return err } if authArtDetails.GetSshAuthHeaders() != nil { return errorutils.CheckErrorf("SSH authentication is not supported in this command") } - ca.authArtDetails = authArtDetails + nc.authArtDetails = authArtDetails return nil } -func (ca *NpmCommand) setJsonOutput() error { - jsonOutput, err := npm.ConfigGet(ca.npmArgs, "json", ca.executablePath) +func (nc *NpmCommand) setJsonOutput() error { + jsonOutput, err := npm.ConfigGet(nc.npmArgs, "json", nc.executablePath) if err != nil { return err } // In case of --json=, the value of json is set to 'true', but the result from the command is not 'true' - ca.jsonOutput = jsonOutput != "false" + nc.jsonOutput = jsonOutput != "false" return nil } -func (ca *NpmCommand) processConfigLine(configLine string) (filteredLine string, err error) { +func (nc *NpmCommand) processConfigLine(configLine string) (filteredLine string, err error) { splitOption := strings.SplitN(configLine, "=", 2) key := strings.TrimSpace(splitOption[0]) validLine := len(splitOption) == 2 && isValidKey(key) if !validLine { if strings.HasPrefix(splitOption[0], "@") { // Override scoped registries (@scope = xyz) - return fmt.Sprintf("%s = %s\n", splitOption[0], ca.registry), nil + return fmt.Sprintf("%s = %s\n", splitOption[0], nc.registry), nil } return } value := strings.TrimSpace(splitOption[1]) if key == "_auth" { - return "", ca.setNpmConfigAuthEnv(value) + return "", nc.setNpmConfigAuthEnv(value) } if strings.HasPrefix(value, "[") && strings.HasSuffix(value, "]") { return addArrayConfigs(key, value), nil @@ -218,11 +218,11 @@ func (ca *NpmCommand) processConfigLine(configLine string) (filteredLine string, return fmt.Sprintf("%s\n", configLine), err } -func (ca *NpmCommand) setNpmConfigAuthEnv(value string) error { +func (nc *NpmCommand) setNpmConfigAuthEnv(value string) error { // Check if the npm version is bigger or equal to 9.3.1 - if ca.npmVersion.Compare(npmVersionForLegacyEnv) <= 0 { + if nc.npmVersion.Compare(npmVersionForLegacyEnv) <= 0 { // Get registry name without the protocol name but including the '//' - registryWithoutProtocolName := ca.registry[strings.Index(ca.registry, "://")+1:] + registryWithoutProtocolName := nc.registry[strings.Index(nc.registry, "://")+1:] // Set "npm_config_//:_auth" environment variable to allow authentication with Artifactory scopedRegistryEnv := fmt.Sprintf(npmConfigAuthEnv, registryWithoutProtocolName) return os.Setenv(scopedRegistryEnv, value) @@ -232,16 +232,16 @@ func (ca *NpmCommand) setNpmConfigAuthEnv(value string) error { return os.Setenv(npmLegacyConfigAuthEnv, value) } -func (ca *NpmCommand) prepareConfigData(data []byte) ([]byte, error) { +func (nc *NpmCommand) prepareConfigData(data []byte) ([]byte, error) { var filteredConf []string - configString := string(data) + "\n" + ca.npmAuth + configString := string(data) + "\n" + nc.npmAuth scanner := bufio.NewScanner(strings.NewReader(configString)) for scanner.Scan() { currOption := scanner.Text() if currOption == "" { continue } - filteredLine, err := ca.processConfigLine(currOption) + filteredLine, err := nc.processConfigLine(currOption) if err != nil { return nil, errorutils.CheckError(err) } @@ -253,88 +253,87 @@ func (ca *NpmCommand) prepareConfigData(data []byte) ([]byte, error) { return nil, errorutils.CheckError(err) } - filteredConf = append(filteredConf, "json = ", strconv.FormatBool(ca.jsonOutput), "\n") - filteredConf = append(filteredConf, "registry = ", ca.registry, "\n") + filteredConf = append(filteredConf, "json = ", strconv.FormatBool(nc.jsonOutput), "\n") + filteredConf = append(filteredConf, "registry = ", nc.registry, "\n") return []byte(strings.Join(filteredConf, "")), nil } -func (ca *NpmCommand) CreateTempNpmrc() error { - log.Debug("Creating project .npmrc file.") - data, err := npm.GetConfigList(ca.npmArgs, ca.executablePath) +func (nc *NpmCommand) CreateTempNpmrc() error { + data, err := npm.GetConfigList(nc.npmArgs, nc.executablePath) if err != nil { return err } - configData, err := ca.prepareConfigData(data) + configData, err := nc.prepareConfigData(data) if err != nil { return errorutils.CheckError(err) } - if err = removeNpmrcIfExists(ca.workingDirectory); err != nil { + if err = removeNpmrcIfExists(nc.workingDirectory); err != nil { return err } - - return errorutils.CheckError(os.WriteFile(filepath.Join(ca.workingDirectory, npmrcFileName), configData, 0600)) + log.Debug("Creating temporary .npmrc file.") + return errorutils.CheckError(os.WriteFile(filepath.Join(nc.workingDirectory, npmrcFileName), configData, 0755)) } -func (ca *NpmCommand) Run() (err error) { - if err = ca.PreparePrerequisites(ca.repo); err != nil { +func (nc *NpmCommand) Run() (err error) { + if err = nc.PreparePrerequisites(nc.repo); err != nil { return } defer func() { - err = errors.Join(err, ca.restoreNpmrcFunc()) + err = errors.Join(err, nc.restoreNpmrcFunc()) }() - if err = ca.CreateTempNpmrc(); err != nil { + if err = nc.CreateTempNpmrc(); err != nil { return } - if err = ca.prepareBuildInfoModule(); err != nil { + if err = nc.prepareBuildInfoModule(); err != nil { return } - err = ca.collectDependencies() + err = nc.collectDependencies() return } -func (ca *NpmCommand) prepareBuildInfoModule() error { +func (nc *NpmCommand) prepareBuildInfoModule() error { var err error - if ca.collectBuildInfo { - ca.collectBuildInfo, err = ca.buildConfiguration.IsCollectBuildInfo() + if nc.collectBuildInfo { + nc.collectBuildInfo, err = nc.buildConfiguration.IsCollectBuildInfo() if err != nil { return err } } // Build-info should not be created when installing a single package (npm install ). - if ca.collectBuildInfo && len(filterFlags(ca.npmArgs)) > 0 { + if nc.collectBuildInfo && len(filterFlags(nc.npmArgs)) > 0 { log.Info("Build-info dependencies collection is not supported for installations of single packages. Build-info creation is skipped.") - ca.collectBuildInfo = false + nc.collectBuildInfo = false } - buildName, err := ca.buildConfiguration.GetBuildName() + buildName, err := nc.buildConfiguration.GetBuildName() if err != nil { return err } - buildNumber, err := ca.buildConfiguration.GetBuildNumber() + buildNumber, err := nc.buildConfiguration.GetBuildNumber() if err != nil { return err } buildInfoService := utils.CreateBuildInfoService() - npmBuild, err := buildInfoService.GetOrCreateBuildWithProject(buildName, buildNumber, ca.buildConfiguration.GetProject()) + npmBuild, err := buildInfoService.GetOrCreateBuildWithProject(buildName, buildNumber, nc.buildConfiguration.GetProject()) if err != nil { return errorutils.CheckError(err) } - ca.buildInfoModule, err = npmBuild.AddNpmModule(ca.workingDirectory) + nc.buildInfoModule, err = npmBuild.AddNpmModule(nc.workingDirectory) if err != nil { return errorutils.CheckError(err) } - ca.buildInfoModule.SetCollectBuildInfo(ca.collectBuildInfo) - if ca.buildConfiguration.GetModule() != "" { - ca.buildInfoModule.SetName(ca.buildConfiguration.GetModule()) + nc.buildInfoModule.SetCollectBuildInfo(nc.collectBuildInfo) + if nc.buildConfiguration.GetModule() != "" { + nc.buildInfoModule.SetName(nc.buildConfiguration.GetModule()) } return nil } -func (ca *NpmCommand) collectDependencies() error { - ca.buildInfoModule.SetNpmArgs(append([]string{ca.cmdName}, ca.npmArgs...)) - return errorutils.CheckError(ca.buildInfoModule.Build()) +func (nc *NpmCommand) collectDependencies() error { + nc.buildInfoModule.SetNpmArgs(append([]string{nc.cmdName}, nc.npmArgs...)) + return errorutils.CheckError(nc.buildInfoModule.Build()) } // Gets a config with value which is an array @@ -362,7 +361,7 @@ func removeNpmrcIfExists(workingDirectory string) error { return errorutils.CheckError(err) } - log.Debug("Removing Existing .npmrc file") + log.Debug("Removing existing .npmrc file") return errorutils.CheckError(os.Remove(filepath.Join(workingDirectory, npmrcFileName))) } diff --git a/artifactory/commands/transferconfig/transferconfig.go b/artifactory/commands/transferconfig/transferconfig.go index c0e1da318..796e99c86 100644 --- a/artifactory/commands/transferconfig/transferconfig.go +++ b/artifactory/commands/transferconfig/transferconfig.go @@ -347,13 +347,11 @@ func (tcc *TransferConfigCommand) exportSourceArtifactory() (string, func() erro } // Do export - trueValue := true - falseValue := false exportParams := services.ExportParams{ ExportPath: exportPath, - IncludeMetadata: &falseValue, + IncludeMetadata: clientutils.Pointer(false), Verbose: &tcc.verbose, - ExcludeContent: &trueValue, + ExcludeContent: clientutils.Pointer(true), } cleanUp := func() error { return fileutils.RemoveTempDir(exportPath) } if err = tcc.SourceArtifactoryManager.Export(exportParams); err != nil { diff --git a/artifactory/commands/transferconfigmerge/transferconfigmerge_test.go b/artifactory/commands/transferconfigmerge/transferconfigmerge_test.go index b18a52d27..48d184c83 100644 --- a/artifactory/commands/transferconfigmerge/transferconfigmerge_test.go +++ b/artifactory/commands/transferconfigmerge/transferconfigmerge_test.go @@ -3,6 +3,7 @@ package transferconfigmerge import ( "github.com/jfrog/jfrog-client-go/access/services" artifactoryServices "github.com/jfrog/jfrog-client-go/artifactory/services" + clientutils "github.com/jfrog/jfrog-client-go/utils" "github.com/stretchr/testify/assert" "strings" "testing" @@ -62,17 +63,15 @@ func createProjects(sameKey, sameName, sameDescription, sameAdmin, sameQuotaByte if !sameDescription { targetDescription = sourceDescription + "Target" } - trueValue := true - falseValue := false if !sameAdmin { - targetAdmin.ManageMembers = &trueValue - targetAdmin.IndexResources = &trueValue + targetAdmin.ManageMembers = clientutils.Pointer(true) + targetAdmin.IndexResources = clientutils.Pointer(true) } - var sourceSoftLimit = &falseValue - var targetSoftLimit = &falseValue + var sourceSoftLimit = clientutils.Pointer(false) + var targetSoftLimit = clientutils.Pointer(false) if !sameSoftLimit { - targetSoftLimit = &trueValue + targetSoftLimit = clientutils.Pointer(true) } if !sameQuotaBytes { targetQuotaBytes += 125 @@ -83,24 +82,21 @@ func createProjects(sameKey, sameName, sameDescription, sameAdmin, sameQuotaByte } func TestCompareInterfaces(t *testing.T) { - trueValue := true - falseValue := false - first := artifactoryServices.DockerRemoteRepositoryParams{} first.RemoteRepositoryBaseParams = artifactoryServices.RemoteRepositoryBaseParams{Password: "ppppp"} first.Key = "string1" - first.BlackedOut = &trueValue - first.AssumedOfflinePeriodSecs = 1111 + first.BlackedOut = clientutils.Pointer(true) + first.AssumedOfflinePeriodSecs = clientutils.Pointer(1111) first.Environments = []string{"111", "aaa"} - first.ContentSynchronisation = &artifactoryServices.ContentSynchronisation{Enabled: &trueValue} + first.ContentSynchronisation = &artifactoryServices.ContentSynchronisation{Enabled: clientutils.Pointer(true)} second := artifactoryServices.DockerRemoteRepositoryParams{} second.RemoteRepositoryBaseParams = artifactoryServices.RemoteRepositoryBaseParams{Password: "sssss"} second.Key = "string2" - second.BlackedOut = &falseValue - second.AssumedOfflinePeriodSecs = 2222 + second.BlackedOut = clientutils.Pointer(false) + second.AssumedOfflinePeriodSecs = clientutils.Pointer(2222) second.Environments = []string{"222", "bbb"} - second.ContentSynchronisation = &artifactoryServices.ContentSynchronisation{Enabled: &falseValue} + second.ContentSynchronisation = &artifactoryServices.ContentSynchronisation{Enabled: clientutils.Pointer(false)} diff, err := compareInterfaces(first, second, filteredRepoKeys...) assert.NoError(t, err) diff --git a/artifactory/commands/utils/npmcmdutils.go b/artifactory/commands/utils/npmcmdutils.go index 883fa88d2..4ed2ea571 100644 --- a/artifactory/commands/utils/npmcmdutils.go +++ b/artifactory/commands/utils/npmcmdutils.go @@ -118,18 +118,20 @@ func ExtractNpmOptionsFromArgs(args []string) (detailedSummary, xrayScan bool, s // If there is no file at filePath, a backup file won't be created, and the restore function will delete the file at filePath. func BackupFile(filePath, backupFileName string) (restore func() error, err error) { fileInfo, err := os.Stat(filePath) - if err != nil { + if errorutils.CheckError(err) != nil { if os.IsNotExist(err) { - return createRestoreFileFunc(filePath, backupFileName), nil + restore = createRestoreFileFunc(filePath, backupFileName) + err = nil } - return nil, errorutils.CheckError(err) + return } if err = cloneFile(filePath, backupFileName, fileInfo.Mode()); err != nil { - return nil, err + return } log.Debug("The file", filePath, "was backed up successfully to", backupFileName) - return createRestoreFileFunc(filePath, backupFileName), nil + restore = createRestoreFileFunc(filePath, backupFileName) + return } func cloneFile(origFile, newName string, fileMode os.FileMode) (err error) { diff --git a/artifactory/utils/dependenciesutils.go b/artifactory/utils/dependenciesutils.go index 91fa38cd1..bb1d12850 100644 --- a/artifactory/utils/dependenciesutils.go +++ b/artifactory/utils/dependenciesutils.go @@ -215,7 +215,7 @@ func DownloadDependency(artDetails *config.ServerDetails, downloadPath, targetPa if err = errorutils.CheckResponseStatus(resp, http.StatusOK); err != nil { return err } - err = coreutils.SetPermissionsRecursively(tempDirPath, 0700) + err = coreutils.SetPermissionsRecursively(tempDirPath, 0755) if err != nil { return err } diff --git a/general/envsetup/envsetup.go b/general/envsetup/envsetup.go index fa2fafe32..cbe844133 100644 --- a/general/envsetup/envsetup.go +++ b/general/envsetup/envsetup.go @@ -47,8 +47,6 @@ const ( " jf docker scan :" ) -var trueValue = true - type EnvSetupCommand struct { registrationURL string // In case encodedConnectionDetails were provided - we have a registered user that was invited to the platform. @@ -268,7 +266,7 @@ func GenerateNewLongTermRefreshableAccessToken(server *config.ServerDetails) (er func createLongExpirationRefreshableTokenParams() *services.CreateTokenParams { params := services.CreateTokenParams{} params.ExpiresIn = nonExpiredTokenValue - params.Refreshable = &trueValue + params.Refreshable = clientUtils.Pointer(true) params.Audience = "*@*" return ¶ms } diff --git a/general/invite/invite.go b/general/invite/invite.go index 4b8316d9f..84bdda983 100644 --- a/general/invite/invite.go +++ b/general/invite/invite.go @@ -6,6 +6,7 @@ import ( "github.com/jfrog/jfrog-client-go/access" accessservices "github.com/jfrog/jfrog-client-go/access/services" "github.com/jfrog/jfrog-client-go/artifactory/services" + clientutils "github.com/jfrog/jfrog-client-go/utils" "github.com/jfrog/jfrog-client-go/utils/log" "strings" @@ -73,20 +74,18 @@ func (ic *InviteCommand) Run() (err error) { } func (ic *InviteCommand) createNewInvitedUser() *services.User { - var trueValue = true - var falseValue = false userDetails := services.User{} // Parameters "name" and "email" should both be with the email value for internal reasons in access. userDetails.Email = ic.invitedEmail userDetails.Name = ic.invitedEmail // Random valid password - information won't be used in access. userDetails.Password = "Password1!" - userDetails.Admin = &trueValue - userDetails.ShouldInvite = &trueValue + userDetails.Admin = clientutils.Pointer(true) + userDetails.ShouldInvite = clientutils.Pointer(true) userDetails.Source = accessservices.InviteCliSourceName - userDetails.ProfileUpdatable = &trueValue - userDetails.DisableUIAccess = &falseValue - userDetails.InternalPasswordDisabled = &falseValue + userDetails.ProfileUpdatable = clientutils.Pointer(true) + userDetails.DisableUIAccess = clientutils.Pointer(false) + userDetails.InternalPasswordDisabled = clientutils.Pointer(false) return &userDetails } diff --git a/go.mod b/go.mod index e306e4516..da2539daf 100644 --- a/go.mod +++ b/go.mod @@ -18,7 +18,7 @@ require ( github.com/jfrog/jfrog-client-go v1.32.3 github.com/magiconair/properties v1.8.7 github.com/manifoldco/promptui v0.9.0 - github.com/owenrumney/go-sarif/v2 v2.2.0 + github.com/owenrumney/go-sarif/v2 v2.2.2 github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 github.com/spf13/viper v1.16.0 github.com/stretchr/testify v1.8.4 @@ -37,7 +37,7 @@ require ( github.com/BurntSushi/toml v1.3.2 // indirect github.com/CycloneDX/cyclonedx-go v0.7.2 // indirect github.com/Microsoft/go-winio v0.6.1 // indirect - github.com/ProtonMail/go-crypto v0.0.0-20230828082145-3c4c8a2d2371 // indirect + github.com/ProtonMail/go-crypto v0.0.0-20230923063757-afb1ddc0824c // indirect github.com/VividCortex/ewma v1.2.0 // indirect github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d // indirect github.com/acomagu/bufpipe v1.0.4 // indirect @@ -94,3 +94,7 @@ require ( gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/warnings.v0 v0.1.2 // indirect ) + +replace github.com/jfrog/jfrog-client-go => github.com/jfrog/jfrog-client-go v1.28.1-0.20230928142526-622034e3f57b + +replace github.com/jfrog/build-info-go => github.com/jfrog/build-info-go v1.8.9-0.20230928084830-478bd49f5d3e diff --git a/go.sum b/go.sum index 8733c3d81..31b7169b1 100644 --- a/go.sum +++ b/go.sum @@ -47,8 +47,8 @@ github.com/CycloneDX/cyclonedx-go v0.7.2/go.mod h1:K2bA+324+Og0X84fA8HhN2X066K7B github.com/Microsoft/go-winio v0.5.2/go.mod h1:WpS1mjBmmwHBEWmogvA2mj8546UReBk4v8QkMxJ6pZY= github.com/Microsoft/go-winio v0.6.1 h1:9/kr64B9VUZrLm5YYwbGtUJnMgqWVOdUAXu6Migciow= github.com/Microsoft/go-winio v0.6.1/go.mod h1:LRdKpFKfdobln8UmuiYcKPot9D2v6svN5+sAH+4kjUM= -github.com/ProtonMail/go-crypto v0.0.0-20230828082145-3c4c8a2d2371 h1:kkhsdkhsCvIsutKu5zLMgWtgh9YxGCNAw8Ad8hjwfYg= -github.com/ProtonMail/go-crypto v0.0.0-20230828082145-3c4c8a2d2371/go.mod h1:EjAoLdwvbIOoOQr3ihjnSoLZRtE8azugULFRteWMNc0= +github.com/ProtonMail/go-crypto v0.0.0-20230923063757-afb1ddc0824c h1:kMFnB0vCcX7IL/m9Y5LO+KQYv+t1CQOiFe6+SV2J7bE= +github.com/ProtonMail/go-crypto v0.0.0-20230923063757-afb1ddc0824c/go.mod h1:EjAoLdwvbIOoOQr3ihjnSoLZRtE8azugULFRteWMNc0= github.com/VividCortex/ewma v1.2.0 h1:f58SaIzcDXrSy3kWaHNvuJgJ3Nmz59Zji6XoJR/q1ow= github.com/VividCortex/ewma v1.2.0/go.mod h1:nz4BbCtbLyFDeC9SUHbtcT5644juEuWfUAUnGx7j5l4= github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d h1:licZJFw2RwpHMqeKTCYkitsPqHNxTmd4SNR5r94FGM8= @@ -195,14 +195,14 @@ github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 h1:BQSFePA1RWJOl github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99/go.mod h1:1lJo3i6rXxKeerYnT8Nvf0QmHCRC1n8sfWVwXF2Frvo= github.com/jedib0t/go-pretty/v6 v6.4.7 h1:lwiTJr1DEkAgzljsUsORmWsVn5MQjt1BPJdPCtJ6KXE= github.com/jedib0t/go-pretty/v6 v6.4.7/go.mod h1:Ndk3ase2CkQbXLLNf5QDHoYb6J9WtVfmHZu9n8rk2xs= -github.com/jfrog/build-info-go v1.9.10 h1:uXnDLVxpqxoAMpXcki00QaBB+M2BoGMMpHODPkmmYOY= -github.com/jfrog/build-info-go v1.9.10/go.mod h1:ujJ8XQZMdT2tMkLSMJNyDd1pCY+duwHdjV+9or9FLIg= +github.com/jfrog/build-info-go v1.8.9-0.20230928084830-478bd49f5d3e h1:tWNlQScbapCz5/EBc+lKBBQcZ/3QLgM3tM3HBEtxCTs= +github.com/jfrog/build-info-go v1.8.9-0.20230928084830-478bd49f5d3e/go.mod h1:ujJ8XQZMdT2tMkLSMJNyDd1pCY+duwHdjV+9or9FLIg= github.com/jfrog/gofrog v1.3.0 h1:o4zgsBZE4QyDbz2M7D4K6fXPTBJht+8lE87mS9bw7Gk= github.com/jfrog/gofrog v1.3.0/go.mod h1:IFMc+V/yf7rA5WZ74CSbXe+Lgf0iApEQLxRZVzKRUR0= github.com/jfrog/jfrog-apps-config v1.0.1 h1:mtv6k7g8A8BVhlHGlSveapqf4mJfonwvXYLipdsOFMY= github.com/jfrog/jfrog-apps-config v1.0.1/go.mod h1:8AIIr1oY9JuH5dylz2S6f8Ym2MaadPLR6noCBO4C22w= -github.com/jfrog/jfrog-client-go v1.32.3 h1:B2M8Gu8EMrokbHWPPDgN1b7YRWwf0oe746epvQASK6c= -github.com/jfrog/jfrog-client-go v1.32.3/go.mod h1:UewnwkIf/77HzBgwCPzOHZCK6V/Nw5/JwdzN/tRb4aU= +github.com/jfrog/jfrog-client-go v1.28.1-0.20230928142526-622034e3f57b h1:WXxRXkWaxFRdMechZJ0pkzZWCI6HNqlxLzAfr3qXjP8= +github.com/jfrog/jfrog-client-go v1.28.1-0.20230928142526-622034e3f57b/go.mod h1:AePTNv5H1YSGycxiL+1jXHCzqu3rCGruVP7S0N+BEEo= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= github.com/kevinburke/ssh_config v1.2.0 h1:x584FjTGwHzMwvHx18PXxbBVzfnxogHaAReU4gf13a4= @@ -253,8 +253,8 @@ github.com/nwaples/rardecode v1.1.0 h1:vSxaY8vQhOcVr4mm5e8XllHWTiM4JF507A0Katqw7 github.com/nwaples/rardecode v1.1.0/go.mod h1:5DzqNKiOdpKKBH87u8VlvAnPZMXcGRhxWkRpHbbfGS0= github.com/onsi/gomega v1.27.10 h1:naR28SdDFlqrG6kScpT8VWpu1xWY5nJRCF3XaYyBjhI= github.com/owenrumney/go-sarif v1.1.1/go.mod h1:dNDiPlF04ESR/6fHlPyq7gHKmrM0sHUvAGjsoh8ZH0U= -github.com/owenrumney/go-sarif/v2 v2.2.0 h1:1DmZaijK0HBZCR1fgcDSGa7VzYkU9NDmbZ7qC2QfUjE= -github.com/owenrumney/go-sarif/v2 v2.2.0/go.mod h1:MSqMMx9WqlBSY7pXoOZWgEsVB4FDNfhcaXDA1j6Sr+w= +github.com/owenrumney/go-sarif/v2 v2.2.2 h1:x2acaiiAW9hu+78wbEYBRGLk5nRtHmkv7HeUsKvblwc= +github.com/owenrumney/go-sarif/v2 v2.2.2/go.mod h1:MSqMMx9WqlBSY7pXoOZWgEsVB4FDNfhcaXDA1j6Sr+w= github.com/pelletier/go-toml/v2 v2.0.8 h1:0ctb6s9mE31h0/lhu+J6OPmVeDxJn+kYnJc2jZR9tGQ= github.com/pelletier/go-toml/v2 v2.0.8/go.mod h1:vuYfssBdrU2XDZ9bYydBu6t+6a6PYNcZljzZR9VXg+4= github.com/pierrec/lz4/v4 v4.1.2 h1:qvY3YFXRQE/XB8MlLzJH7mSzBs74eA2gg52YTk6jUPM= diff --git a/xray/commands/audit/scarunner.go b/xray/commands/audit/scarunner.go index f78ac8dc2..7a364076d 100644 --- a/xray/commands/audit/scarunner.go +++ b/xray/commands/audit/scarunner.go @@ -135,7 +135,7 @@ func getDirectDependenciesFromTree(dependencyTrees []*xrayCmdUtils.GraphNode) [] func GetTechDependencyTree(params *xrayutils.AuditBasicParams, tech coreutils.Technology) (flatTree *xrayCmdUtils.GraphNode, fullDependencyTrees []*xrayCmdUtils.GraphNode, err error) { logMessage := fmt.Sprintf("Calculating %s dependencies", tech.ToFormal()) - log.Info(logMessage) + log.Info(logMessage + "...") if params.Progress() != nil { params.Progress().SetHeadlineMsg(logMessage) } From 02ff30b2a6e0bfb45b753e60398d720ba38fb521 Mon Sep 17 00:00:00 2001 From: Eyal Delarea Date: Sun, 1 Oct 2023 10:02:10 +0300 Subject: [PATCH 08/16] Export GenerateSarifContentFromResults function (#981) --- xray/utils/resultwriter.go | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/xray/utils/resultwriter.go b/xray/utils/resultwriter.go index c4fd62fdc..9a97a7e87 100644 --- a/xray/utils/resultwriter.go +++ b/xray/utils/resultwriter.go @@ -115,7 +115,7 @@ func (rw *ResultsWriter) PrintScanResults() error { case Json: return PrintJson(rw.results.getXrayScanResults()) case Sarif: - sarifFile, err := rw.generateSarifContentFromResults(false) + sarifFile, err := GenerateSarifContentFromResults(rw.results, rw.isMultipleRoots, rw.includeLicenses, false) if err != nil { return err } @@ -172,21 +172,21 @@ func printMessage(message string) { log.Output("💬" + message) } -func (rw *ResultsWriter) generateSarifContentFromResults(markdownOutput bool) (sarifStr string, err error) { +func GenerateSarifContentFromResults(extendedResults *ExtendedScanResults, isMultipleRoots, includeLicenses, markdownOutput bool) (sarifStr string, err error) { report, err := NewReport() if err != nil { return } - xrayRun, err := rw.convertXrayResponsesToSarifRun(markdownOutput) + xrayRun, err := convertXrayResponsesToSarifRun(extendedResults, isMultipleRoots, includeLicenses, markdownOutput) if err != nil { return } report.Runs = append(report.Runs, xrayRun) - report.Runs = append(report.Runs, rw.results.ApplicabilityScanResults...) - report.Runs = append(report.Runs, rw.results.IacScanResults...) - report.Runs = append(report.Runs, rw.results.SecretsScanResults...) - report.Runs = append(report.Runs, rw.results.SastScanResults...) + report.Runs = append(report.Runs, extendedResults.ApplicabilityScanResults...) + report.Runs = append(report.Runs, extendedResults.IacScanResults...) + report.Runs = append(report.Runs, extendedResults.SecretsScanResults...) + report.Runs = append(report.Runs, extendedResults.SastScanResults...) out, err := json.Marshal(report) if err != nil { @@ -196,13 +196,13 @@ func (rw *ResultsWriter) generateSarifContentFromResults(markdownOutput bool) (s return clientUtils.IndentJson(out), nil } -func (rw *ResultsWriter) convertXrayResponsesToSarifRun(markdownOutput bool) (run *sarif.Run, err error) { - xrayJson, err := rw.convertXrayScanToSimpleJson(true) +func convertXrayResponsesToSarifRun(extendedResults *ExtendedScanResults, isMultipleRoots, includeLicenses, markdownOutput bool) (run *sarif.Run, err error) { + xrayJson, err := convertXrayScanToSimpleJson(extendedResults, isMultipleRoots, includeLicenses, true) if err != nil { return } xrayRun := sarif.NewRunWithInformationURI("JFrog Xray Sca", "https://jfrog.com/xray/") - xrayRun.Tool.Driver.Version = &rw.results.XrayVersion + xrayRun.Tool.Driver.Version = &extendedResults.XrayVersion if len(xrayJson.Vulnerabilities) > 0 || len(xrayJson.SecurityViolations) > 0 { if err = extractXrayIssuesToSarifRun(xrayRun, xrayJson, markdownOutput); err != nil { return @@ -298,18 +298,18 @@ func addResultToSarifRun(issueId, msg, severity string, location *sarif.Location return } -func (rw *ResultsWriter) convertXrayScanToSimpleJson(simplifiedOutput bool) (formats.SimpleJsonResults, error) { - violations, vulnerabilities, licenses := SplitScanResults(rw.results.XrayResults) +func convertXrayScanToSimpleJson(extendedResults *ExtendedScanResults, isMultipleRoots, includeLicenses, simplifiedOutput bool) (formats.SimpleJsonResults, error) { + violations, vulnerabilities, licenses := SplitScanResults(extendedResults.XrayResults) jsonTable := formats.SimpleJsonResults{} if len(vulnerabilities) > 0 { - vulJsonTable, err := PrepareVulnerabilities(vulnerabilities, rw.results, rw.isMultipleRoots, simplifiedOutput) + vulJsonTable, err := PrepareVulnerabilities(vulnerabilities, extendedResults, isMultipleRoots, simplifiedOutput) if err != nil { return formats.SimpleJsonResults{}, err } jsonTable.Vulnerabilities = vulJsonTable } if len(violations) > 0 { - secViolationsJsonTable, licViolationsJsonTable, opRiskViolationsJsonTable, err := PrepareViolations(violations, rw.results, rw.isMultipleRoots, simplifiedOutput) + secViolationsJsonTable, licViolationsJsonTable, opRiskViolationsJsonTable, err := PrepareViolations(violations, extendedResults, isMultipleRoots, simplifiedOutput) if err != nil { return formats.SimpleJsonResults{}, err } @@ -317,7 +317,7 @@ func (rw *ResultsWriter) convertXrayScanToSimpleJson(simplifiedOutput bool) (for jsonTable.LicensesViolations = licViolationsJsonTable jsonTable.OperationalRiskViolations = opRiskViolationsJsonTable } - if rw.includeLicenses { + if includeLicenses { licJsonTable, err := PrepareLicenses(licenses) if err != nil { return formats.SimpleJsonResults{}, err @@ -329,7 +329,7 @@ func (rw *ResultsWriter) convertXrayScanToSimpleJson(simplifiedOutput bool) (for } func (rw *ResultsWriter) convertScanToSimpleJson() (formats.SimpleJsonResults, error) { - jsonTable, err := rw.convertXrayScanToSimpleJson(false) + jsonTable, err := convertXrayScanToSimpleJson(rw.results, rw.isMultipleRoots, rw.includeLicenses, false) if err != nil { return formats.SimpleJsonResults{}, err } From 672e9035c32af7d1b6800b3cd3654e25c68ab3b5 Mon Sep 17 00:00:00 2001 From: Asaf Gabai <77976014+asafgabai@users.noreply.github.com> Date: Mon, 2 Oct 2023 11:32:29 +0300 Subject: [PATCH 09/16] Build flat dependency trees while auditing Gradle projects (#976) * Upgraded to the new gradle-dep-tree version, to make the scan faster and more memory-efficient. --- buildscripts/download-jars.sh | 2 +- .../audit/sca/java/gradle-dep-tree.jar | Bin 10000 -> 11855 bytes xray/commands/audit/sca/java/gradle.go | 91 +----------------- xray/commands/audit/sca/java/gradle_test.go | 89 ++--------------- xray/commands/audit/sca/java/javautils.go | 75 ++++++++++++++- .../commands/audit/sca/java/javautils_test.go | 68 +++++++++++++ 6 files changed, 151 insertions(+), 174 deletions(-) create mode 100644 xray/commands/audit/sca/java/javautils_test.go diff --git a/buildscripts/download-jars.sh b/buildscripts/download-jars.sh index e5e6a5510..3e9f10a89 100755 --- a/buildscripts/download-jars.sh +++ b/buildscripts/download-jars.sh @@ -7,7 +7,7 @@ # https://github.com/jfrog/maven-dep-tree # Once you have updated the versions mentioned below, please execute this script from the root directory of the jfrog-cli-core to ensure the JAR files are updated. -GRADLE_DEP_TREE_VERSION="2.2.0" +GRADLE_DEP_TREE_VERSION="3.0.0" MAVEN_DEP_TREE_VERSION="1.0.0" curl -fL https://releases.jfrog.io/artifactory/oss-release-local/com/jfrog/gradle-dep-tree/${GRADLE_DEP_TREE_VERSION}/gradle-dep-tree-${GRADLE_DEP_TREE_VERSION}.jar -o xray/commands/audit/sca/java/gradle-dep-tree.jar diff --git a/xray/commands/audit/sca/java/gradle-dep-tree.jar b/xray/commands/audit/sca/java/gradle-dep-tree.jar index 2762a71ba29b1c6f99fa94a823dd7aaa97c22e4e..532445bc1a337a6c2055c4b31d255a80f891ecaa 100644 GIT binary patch delta 10035 zcmZXa1xz2?*7k9S7I$|j?(XjHP~6>haOaP^ySqc7#frPTJCq{Du zznMG_$$;_~HIv(GHJebtz-ow-QJ@$BczuFncOc%tBh9>HNfhFXTwr2nUz^z~xDIDJ z#l<0WOo9j~#n>}2F>P~ks)iz|%Q~CWy`CU$p55WRjN0Lh#>FJWY|+_=$3DyObhFGl z+3+-xNT$95wD^&oDWF5q{u(gh`TeTrtF<7o`wSu~LMbQ<%yNE9fXF@mW`XM8%9Qw>;E0w@oSU@Ict zC6QkBgx(ASdqn}JwKp-!H}W2;n-5xHqf#KP7pN=%N&&=#_+G4n`uXo#bHEHR_5+~3 z+XjDzF~IJ5mnvL_WS33Hc(w=xMXnMTa+$5g$x`_!7^67h`eNL}09L`{Zx;h3^x*d~ ztb%n37C)v_5vu3vZd^H14)DTkGd<)9^K%6SeV<3XtS;{ECKm2iCNB8ZnB64#>i-P4 zJh}iYvbfiWB4t&H;u|e39USFDbG&Lp7s`*b^T}f9rt;lCsij1RrG^zhg^89&#K%WS z+9w*9Wtn~Hu^TL(12kFfBC1C@S-U+U#7p5qK&_$S>$oN#Gg3} zlC zl+ToEQEWbgCN?x%c5AqOC4q^F5T!Ql2&+1gbGob{>aMdJVv6`vpN&HzhGvcBVsdh4 zlJ3#M&UelmfJTDl8aD(q-F6xSQOR)+Zu1r*3=P>S+$ig^OgCv2M`b6iKJ%Fr9tRSa@E4PNu-a5RWDeK8 z+hxm7?di7rd>ia=3^{94(sE{MQ^gl*uCKRr9VBN}<)$k)#yqPwe!lz&)yp-JZ$I1+ z08(VFZPB#owkc)XT${%TXEFl<_vk!b(#JM8id~oyTFPwlXJo|e)jE9h18Z5$N(Lj{E}!F*&dCv~xe;Mi@69Jxequ1A z;V-x;DDaQOVoO{7JylnNhCN70Co~el28e~N##t*RN}zP#QeDt_WaLg$q&|!19(l|j zlFqdCtTJai+i=NMx!?QaWc7ncQ+tzp5q%G5Xi|wVi8Q@Kr8J24$q|S(o^T#$`0H-S z`3vWwari24FqEoaai(bqG`-W}>G{iV-~;qaJ=q+LUJ$ONvY<<`=hGZQxKgZXfo}yj zG6ABc`33oI?A5BAnPN9q|hXs!#r z%ycAm!#%;UbPYpPQ9My;A~0H4B0Qa0RpU2(ylBXGHor9MhBpfWJ4eoCc{wBdpRKf& z@J^=L+6_U@g@uZ;(fFK$!NUa6=&f1dc{8F+C4@GCS$u+IZXA*f~CfMy-Hr`?b+ zf#ZF|^-wtGiW41?9=;KTdz;zQw@Q~cEw;}*og2jxA{_|$X(|3tk8w4Y9#|X9tVHNI zTI{Z>->p_1qoH|xFC=$cw>ji~A?xrGN04-CWv_Yra~enFK1I@Y9^`>UjW*B1N!kvG zaX2*yVdT60Bo0!Y%tk}p0n4Z;#3l*k17wD*C>*p9F@pTGl*eIn20$Ue?Qk=vDSf00 zCr{5szK-ce*ssc#qbL>Qp ziP-$hAmhn}Pee{hgWCK_4NCZwdTPV_+%uOEAcCWAL=Wesw$9{3hSa$2k?0I#5%W>U z%9F_X@W`9}W+_FX1aE&Vc@Jv%A~bc|G`r4n)0e1R>WJNuOAH9vE`#jJE=$Yd(cL6J zWQR%qsd0K#zAODRVB4r5l!J~TNk!P0q~l+zC(M&mBmmzR)(W_d~Rj(Ch- zFDzb%KTTfEN<~Fdaf2@FZ9zuN3X%Q`-bM>p`w0EGM=Lg}09tM6zW&kfu0S%ga;$we zvCmcDs#td-xaJ*X&eSyfb@3}k=WhSPPv zDU@_8uubl$whbApONuz9^Uma~*myyBIua0}^b31sQjtja41q9UQlm6h_U}t081ke} z-S7USyBX!k=dJ>KSuQxY`9_kTW`#>Rz7JMyc4gQCm`$M9^yXk5RwcJD>@y04e1KWx zO5iq2)QbAYGo0z%`6_!D6lXmtR3S%Eq6^fksCiCnRH26io4Y}{^S@CqIr^2pG$ETP zK1GZwiK$Bsw7{Cf#;xJ8i`igqD#FW}z>avjU#rpQ7Gtj{3_V`{m(%wJM1a#IADXiOV)CE=-=;IpjS& z|2U`?n=U=aG1)g-BH$?u;Kt7iutV|>==k1?7&S`$=FMm=kMAuVAv&dPptf4SJ#bH8 z-X^+UX-|6XL2q%fd-^Qd!&sZu_JS9ICx%CFrrIg47j5^Ftn)NnaA?XPT)M>nG07{M~c!74(bqo@^HoEgzs?xK4gKyux1f zS%&7gfg{8O-uRxr*pcMB1l@smUbWxK ztQnpHAz2tVv@Odp_jOVjXWv8+yfF)V=C@^H<8P4njKjM0+Mt{=U!C@kg6j)07L?3K zwl5FJ3O#=da_=D!`pukn85KPR@pHLZZp<+6Y`!5aa{62*Y!|fii`lfdAu1j zc-|I~PYyG?)pyFc6F^67v>I?U-`2~QyI7Z|iav?kLmX;ESOcpv7EQO>e~W-@P`HcU zX-3o_@ronw&B=tQ)k)?Gu1+OXz+=-)?2jOc)z!si4K+v-i-`L) zBSP^bQ<{-wvsywN2eqo4=$d{Z9aetVfE}*LH2nxVz zb-@Q&-1I1RgAK4q*tkD}R$G1A>p4ldMAJ9b5-7~pjE|S5m90|Ee(Xdk_`GUd-Rk64 zUlv!Q*PZv*mQu%u^6mVJ@3JOL3RXTow%-!WS2!ky{6)GDa`MLOpGsVcpy-^XpdVS+ z6>Ru(&-M=SZtMf%-(B(Foi13g2eJeN7+5nD7#Kz3Hde9`DlDMrXlBl6YG>@~>aVKc zI4}I!Kl@X;b55Czj2wJR7(1m*AxxBBm40-fn0lyc^vGmm`>14H96>kQM--&AIwr|A zIS2UsXV@Qhs>d>mbz8VC%_ z>=$eU=&bS4@ZXu>+}h2%u%CA}Tz#KyryK0vlYl+Y5Fv1T3%{F~VB)S)a4oGB1+BMC}TBSRY0Ix|B%=?|LEk4D z7QYN;dlKC1H%ncqFvaLyok*O<(M_z}`F3jPhf_fQ=FAxMS>W0osXBCMl6H0!&f;JW zvdW$bT4pLF#Q}P?&*9()yY$&O!N?TaXGzCr8XJc>d!8CqCIeEAMk(ZK^Rw0LCUIQ7 zqEJVMsBz=zEusJ1b;gVHe2d^{1S9d!% z*MIH8T-9-hd0`9z^yAr*GHWp!hWI2hENfve@NUh`l;JX{%1XL6bW3J*eZH*N#AhuM z71G8X$VZyNo_%)qA&AN*_J=9|9M`(5*VjL9uz|;3E5gtZ=QO4<@vt0jHY->(+3c+l zwF8Z>6d~GFD$EU%)U!aHFr|!?4c*K{a^|A*Z%IWOHR2PRL{2u@ov$q24_tG`8F?`K z4^(l%%aS0IKcWiitg13zOVr_;{-QO%PQ};#!RxzSMVnA)`X+c)-Y(oKCK?HTMu(LZ z#wwM}`sG89--l4;=8_ee7B3`bSv2ONO%^)@eH%K>2=Grlv~vOa?tXoLTh7You}y4} za=^X(r8vu*R#A}eXm(k&H>YT57-!&$9w(BOsoXQie`PTX_>FZ4`fdDHk(eAtJA=^s zcz=SZ1`fUvf;6le6_^@Dr<=uz!Fl2vgvoNg%X|0nyVWTl;OdC9&Aq@}52do7+9ib1 z-#5zU zDST_RxZ^R?XY$Zp%`(ZJklJb5K|D$o4+1!uvRqJQvLaw@-XTo~AI(O6Ak zZWOK<9B;n-AA|1iqbOIm?!y-z3~Uht3@ix)AMpLEriJ^l$vN4h;AjosGSir23&-|J=-X6}Ojb!D32@+d9B|CH{nWetz)el9MC^<4!$;xu3A)6hWA$-4eaXsn0a)Z11X?|`Zpm@zhEYo|11an27W$fQ>~?4(zF@B@uk z{i7ho!gr;QV3|)K6WN9`a{~H4XFu6U zpbN^>OC5>Yj9)VtOprW$dh9W0=G`0DC_$MB6nSdxXkaf3sY5P%UGTPGE$!2WhvP-`D!tDJl0Ln2z8T{oN3zC0`Oy#(XjLZ#f@l@c? z@d!ft!;Kmsal6}owknH%*Ny)(=9&(-t-C|7N#NG2%Ro1kZ8Ino{ARr|aaEjny2K zcrAfzlyq(0Up8do!=8dH?kiTu_M|u?Ask|AfF%9Zg94iSR@9o-cuMEXLJTP_+vSzR<{1mYa5_MZNs&Pj zaC3X*;Yy4xfn^~lS1~cw->6N8o^BcAkX7r+){gBN0b87v8tFP59)7!$wA^b#T^8FG z`lnfpr#KbOGD;tH7KA=qG+7f6REHV zhe1VDL;;!4VuSe$<{O-gQf;&&+-mTeWSM9RQ6^n)lw&0hc(p{A&JfT(Terv8-vrH; zcaJm_=TCN};fK8~$6)@z1-~fiHzQ;~IxD}~|B~giJNGd3 z8}k0~IjcbjoudYCHZgYqh@=jC5h6>7qFFCR#a$Gm3y96H@InwOg}t*ioQXsUg++Jn z?w;w(-?hf%g`uG7BL#?XmW$~EK;@ZRtqdHj0fl?%UXnlS)nDpb=2)rz~8FHzz)rTHgiLwXP41G`)s!FSYezBfsmXFvIBDnbVLlMUeW_v>_ToEccb) zr@Vm=7l$|g3`(6nTAwTtvor~>VundxUlLSg1mIOK2|&;L-e{~C)G2SoYFjxODQ*>p zN^NpO6ym6h+8oqu@%%A9ZL07rop>&CEY`mk-br!0lZGkuqENUJ56PDz%WZIysuNg- z7s%}^&DMSQmf|5{pJWh!YPunh_kv43HXHCZ&RyK3g;mudvmw>Wmfb(AVygY%xwcHR zUdLGg54?1CnwcHRrA1e?%`=Civ%f^iHzXtEfc)dI%mKD@f>D0Y`&G_p*sU-IUa6@b z&&7*HP6o77sv~e67ddTb!}o^NX@&^%FLCrwj!JvPL)9~~Yd`BDyhvQ9JYk0!4jyGF z=g78Iq^0Z2;dC2xd#lz?sYh`fZbr>LEzo5nbZQS3vDIClACHG^Z~b;!}0T zk3Fn(HWj_-Dt!%!WNr0B-dXObA@qt97b|%4Dt&!xUX~zC_N$`aeeF3<3~XPfOz~Dn z08OrkBS+o@cd|~vIW)k>_-hLA^I<==Beeww)+ymma3da8CP*wp7!$$1L|0|X8N3wB z#S%JAwTLigCS=Hqwlo|UECg7UqIC_tBRM727e$tH!ol8mR$r&7Z&xB3;B-@QWg+VK z_!c?KJ)zu-!^cfNby8-y(ww_8M04i?1PJh3R%}u3`{4SsBA8u-QxxAIVPKfMj5cGZ zsL`yLAdcv)=V&}(lxX5=jAwlD)BbIiNHd+(C4KeBT#^%>s;>@f7MYJ>h*l|D<$_F! zjbkz74{{!jgYk+Yy3h(d^^s*OG@W@tsEP-LVwrySC28)T{RejT;NQ-_rTZ8$hy9uN zG6p9SYgWE0F{D(3(H<~;mG5tr2P)j}+!TH~|G5Jq*>i?g#dz)TaSKWP9@O$_js;bN zcD#Bky0Q#(5{(4nko#BS(6j?2<~!v3x!YoNV#N6%8Ga!wb4Md)8=%uXOd{B)`r>7W zesyiy@(*_H+_AIhGh%T*+Wl7Ov07&&>ltXVv;niUCJFP*dZV|J2~m?e4&aW_Cwztc zmOFW+NcIjkK=pN1zM;)!Q+VT-{{E>rjwQht{Gw|!vM+;#qvG13>lJa77LYpDQd%b~ z=?lh9THLML@XOXU$+Hupc)lpFwy4TVsCSv_E`fkVr<8u+v3%v+D{U*Mk}PJE0eoe4 zWBS6l4%jfNE4EzHK@tMZ1GSUOv=q&Y3V$&9G~|EEcXPHPBRM!QuqI3}Fxr1LjflxY zREWTk_LHx=7H)u}1r=9jXF_QLt2&LIbq0kzElon9G8{umxOyfSjLGEW1ZC%`*{_2s zLftCu_O-6oH5&$Z+C4PSp|AuK+w*YE^LW8`_;=*g=qpc)X+@%!4DX{2*XbLEtA~JX zo@B1~+nZ`IkDV(KyP?z1;Vfw4uy)pxAyz=&0Bah8F_QHu52+b5QE-eMT*U6EIAf?o zPT~0^bopMF++SOXey{Q$L_pK4cr5*21>6t&rY zxjR3uC}bYOF56=bXgaI`H6J_9=(AWp7V>l`b10MSEoI}L4(dw|yV24$CgHSg#U@}{ zZtqwRV|1RjkN0YzI5#&lV_IIjwXu?Lc%Lao3u1rZhG%o%aI^PjLVE5&<*VKW<%P0g z5ei)L4$O+xWyYfg+3NAN+yglgf=Vz>_k_{A}TLf}k6{bDr@D$son%mRc z27L%Txp`|WiY$JDvg$1JInfO{bpcAcB{rKpmSUc&Xd)Iimc!U{?!0h!#LAh5FDmw$ z=O(Q-a3!>4^a&dH1%;(}P0g8Pb_R=b4+lnS7Czi~5Bhup1bQQTJtnRjZ0Mg{0TCPj z4I;&C!zuHRQ%|@h%1pXK?V%as)+WWXSdLtXJ~Qi(@b2Tb@z$o!6l{hVA)pE-QXEoY zgxi|_@-Tw^!cbX$HKVGQh0tj6Iyiih{ZV@@ck@8lWKfYZ?9sJ~Hd_WVE;f0zRDL?i zZszx3|7<4wt-`)4YG=pcWEyeYsI_kft99y!g2vN~EvOQKd4MJ%z(TMzx}7H{&XwTeo;{MV_XopUjoNh z()zJO&n^JQu&DfE z?>80W=Vl3gS8a{)cOh56X_EOBdPQINB- zJ;}euwKp4BJO^s9>uwS%CAkiyX|<+&Wp8!JbS68|PZv_v!~iVI3F$QWX!0oK2F8~1 zPDg%<+ehwo%_GI#*zieAol0++bTX!rS?am4#cds6A-5}6XN^ad4>?_CDEXS#(0dGF zCGN0sz(-0g{^4xX&TZcCz>y!Tysw#JM6E5Q^O1m%g_D}FtN&Bc#atofPJuW(`#E#E z&E@{Un}rL!S`$z-6%FE0El)RRsMlw=(mR*Y=sh(#ZBiai7pqGq-+e0vO(wq3m1=0# ztW@z4BA&D;84>s=^H8 zgYW7#M8(<+NPK!Ss6iE8zC%9mRa*#8)nQb}-0Sv!F*lXhxRLUr-GhqAzZkzj07;IP zwg=c>VYenN1-3$R^(nK7BjSlkHaNrZkdKYU*VgKPa0U4+UdIT3M_ZYocH(QVHw7Ry zjg!%O8387+?gl*_Y)NVpSz{bur%#xd44EY4FWfHmNxJg{KLX5ugF_v%sxqG$vA3p8 z2aT9RS!vh)yw$7kgTFs7F=A3s7IPfN5S`=l`uZ~PPWuI&m{ZQN^XGnhA=~FsL^#;G zqDBBs!L7=B@2AjMK&@t1TT!BP9d#9~^#CllYc#NBul1n|?_>x4{h?8+S~d?Qg`bed zt1Iv7VgEBK?F+y1?ybwhx42qfl=G7S#7oS-e4#r-a*Y&;efC!@X9tNbmBTeLLg;Bt zkLUWLQlYC23WrOiP{{XjZOXa`jxDq{i7c(<-r)6Tf7UxjbZ$RQJU0%Gl}wEydq#vd6BQSrU%btpQFh<@Sma2fU77G|&e_;-?%~TsjzD zzDU?Lownby@#kc~^fv=2)p)nI0c_kAigM%{am@bE`Q=>OlgRc4B(+xFV7RkJ(ueYL zI$wIf+|by}sm~Xt!0HOCQxx&X*34Z>`#TUj8Lh!|)7nxpSk~oa0Dhw|N8eB>NMMDy zjF`+ztwMB^W;GsU^IX!+O2~D$E z2%jlwNGg4n0Pue+kiVw5lF!IE{+{MaCL@phhqdJF|L}lZ=$}C@Dhj55s7=A~4-+V) z|KU0X-+vEyQBX>f{C`gPU!LUuTO~mkXo)+Re;ebR|2D>z<-j2@ApbR8_ILJwJDUH_ zcEwEG!R7f6Kl8tn|F84)m!$gNRT319oxDLw{!e}XA-ewe6)>=pAa2rs)%Ujs!~W+t w=uqK9jL{~ukmDsMP$B&j%Ks|_LHAck5KCDO8s_g0ApdpML4twB(Ea`NKj`EL+W-In delta 8219 zcmZ9Rbxa)0+Qlgait7T!p}4ziad(%Z#k;uEF23mEt_2pi#hv0s7PsQ=#jTWw$&;~>I6$v_$p=Y3(&jcD%vKnFCA6fYEvn&N;={nK zkatar)kuToE*a(OUoDq*S-aq{Dbl0i&ak2>1?lR9(ol~HrSK`zhTmW_huwtfqAvM0 zbS!EJMEzPjRsVSaI_JEu?RMV{1|w7vq$6>~k|CCT`8fu|q)LFtz2MslMVHte27_L< zq6u>D-mtsF6&za~dICH2dOv(xLI>g$pbLKe;FDs_d6wehGeVf3wAL|ZxfSGmrsY~o z{=3onEE)dmD&rdhwiS>Ddn*`W)YY)p4$&7E=f|ZhLi}a}x&*NltpulF;M-o}y*!1z zuh#C5DlYwi>))FMws4HJR_&3Xj^fxJPpZlP2gNt29E56yv}7=van%%Xk< zdbaO}_NS*GSzVet@3-R_?8ZdTX#gs3E-MIJFzS zG+C|Q^m*|_3A)&c*M7u+l(ESuorcppve5{5coj(ZLY`9Gh0f{KHeMDMilF~ zugm3Z5+OWKuQ^S|OVDh)Y;DX}j=@I$9oxLqsWqU?+~R1{)x}=L{JG{#)81e*tyeRj zmWNG~FrJ|l(}T`oIN3Yj@>12H)nOncSzZ?hICHuNF!!#pB)vlkp4m_1eqrw790Jx2 z$JC)xSC2ngC=;NZhXP@dL=>P|HkrGwxA|Qwil|lSa#pjw;>r5XEcQ-OT~V9zYjf=I zg`YPMWzZ8e%3{@{yE*~5SreRnFo6F%>U93e+dtc6`A(^blDcd$6i}GX<)q z>A?>49mdRR24)(kbQlP|xkqlgY2I`l=S$WSF26c`o2-iY8apioQ36|RuFZY6$K3rLk7~LpoY(`aVQ1|2dvd5Z498I{_hlB|16=R$81R|ei zrU#4<6apoxaCPI9x(cbIH$0^KJIqk8bY19S)_Pf*;(%Fc8J&5}iLd6Eb!J{jk8(0G zVNz3+RYmF5YETm6O)ChferRhW4^jNU)7>zQx0v1_owVM=SvV}Nk@0GaY2_@^bf;X> zVu~}C^)@PxB$&Zn@Vt54Z}m9aQ>>UL-8f=|Fzv0m3OG@)VV-2KbF*adsJu6^aa8*m z)9*(7xY;N^DFtn^8nt3{l&YM6T#--svyVcNxDu_%*0;CL#P%R|*HFJzJrUUeyjtO* zD`UrlTZQ@*ATZBGM@2$Q4^M?LrE0*VP;~qI;aJ1-qjjW2jh(Kn)qXK|YGY1NY2J@| zzA*s?ejd_O_&etQwHbnuvSQ;n$JreMDGv8b81-;B?F91gPSsm>g;AdFBm_c zmrz>coA$Bg?I#f15_22AeGVf+(^ue{T1^Vn3e!YSET*UYp~LxiFsR~z*qf*7Dw)y9 z2kOPuF5*lH-P~Zw>%V4^hsh0sE)ZL82dupNnj0Syfaef3x{A>YD}Km@TO;YY?va@^ zA$?)WTffdg4p-@+%64{@?8F)z4k#R9!M+v%~ORT6JiBZX&@Uj5JWG698}3dOQ$Qb z>;P)Osgn`_fS>kRr?g;ox57a9>B#9#oX*dBnBfhHcmDpO-YsX3>u(K^D3&9QVFTw*>6*tQpif)4u$cE99c=wsk};M9v}yF zWS?hXVCwxm=+0BWp=e^IG?ANLpHS=uYDs&&aQ{smHWqlso2w2KKkl{4_{N=n-Pjp8 zTEWB76|OMSF#s`3iOei-?5z3W>ti^r+)DG&*rvrckwIMv+{EC&CZSHKY4XVd6iFYHTs0FrBxEXZ#8*Octh*}*^u_ZR zD494n_W2FaG^K=IjtH1Op)KdPwi9KT@@<&kGLtC1vB7tAo%RxZJC@WKk*7CO9Mc}n z)DzO@Z+cB$$>W3kEi;yZ0MJ8?A4mUH8+nZyrfQwV$QGD*{4*gR_n z0`075`nupZX_S(=`HO0v{~fy{$aGnJ&CFCa^*b=niY2mCr?)WY3%V$Wej=|_$?eDS z#Hf_q+$c)+&-rZ=YKgyW>GoHxlpob7_>9biHmGLyj@{MW?gji~I<#9riNY*ZqAo=a zC24%;hs?(!{NXi6X<@L#Jz&>Q?b-bbD`sv<5U0C}MTw94Fq=%@s;O8Z=nCbaP3K_N z7LR12joxo+xlc@NGIbPEJ-tT{?|pWQ`NBM`?)5IB$~N+5U;jHXm7q%ZT>U$dKrI(a zoq+-t4tyDw^=-65B=;P24WOr{;Ox|&;RS*H^= zW$a)oCh#Uf#vtNGDAS6*aZ%X8W)BsH$;SrNMhPjB9Nlv_Iv48>5E90cfL7)F(?*E? zZ}S-br3tP1)OZSCl6>j3j#c`}GYgH0)CFBi#TfT_VO}Sv@)j>*(2E*>Lg zWi3Is&Xmc8{<39n+Bb80 zu`X%S7E_3HPR9{h9W;RCnJ4ulkK+@*L=^LJ-OB7N1@}WUm^PKk504Q2j>Yg!OrQFA zGh5x7>qo5c>um?jmg}SBrP0s{|LyNpEKBI_VQKVIvRNI=ew(jok#jA9&{^~%S9I7Xoz@~uh!xY zTRpZsVvHbkApb0YGn~~7fP7xK3l(xRY#y`ITI2>NpAC*0<(NKFA0JFfImd782nB~V zhk%2n0(CorqXnna+skFCa^E1=MpY)Y=D1*;mSQfRzgXLml^Ve)^GwJIW+h)o%HEE)aQ~`yStcnsY&q*X+up!#wV= z&x|eU#<1nqZR4spqav~E2mYEq)~L3usC_+OexuinskOaM6mM#;BiZ`reRGp7?u0~t zDbz{7%?yzTFX?b*07c`n$Z2iyHfsXAAclgZA` zDehUUDh1K(PkH5ude&f`U)PUMrSbIF-*&f?X?3k0>;@S*x8gZwpSEdT%j(Zn6%L0+ zKFw_*@o0kuw*yUO!WP|ss|^Hp^=q*%=4%iqL+Hf=?L(zw3s%(Zor~lhGJp=b>#6=f+u!wA{3kQ_n;K4v$vf=jqZFoZ|j9 zxDBW!v1cVS@x0)9b;Q}*U>O@O`>5XuhxIS>^)*pSOD)jZ-fzY-#zX7XC$~Vv{3p);GFAFkFE0zd5fKnV zkr5E+5;usFDF|LAyV4+mIINv5Jv?%CSAerpfE@TB(6saeB74}>jJCFWaF$A&V)>&s zlNb`NGIOB|q)Sb}{rjjJ%cX``kgVPde5x-BG!r6P9O~%!IO}V?9iqH;IS-5H=aMg+ zx5^`obahv!ab@U{v3&HL9To4;t%>ek&1)#je9Nq2EQcJ?hVR-yV_fD=*`+1ZSYg(k zJQ@|4-p&0FD)@GD5%UE(=^9C5dU=o5+LRt?XV+%tz^~Xg2BiK;!_yBjr zHwGNh&qB-c+XTeG0S#T zdw;YdOH~B&+p&7QGAK)R$%xwerRfmWjYi!e1C;}y!ove-tj*PG(qPj5n1Xf60oseG zPSi@LQIju?>5)8WOOiiky;F7yTN`);;PWfj2hxnN_o44bt0fdm$SE$AByg?D!$5vu z^)cI2_urW1mSV<{M28bYigcsY`1FB`+O2?f=4a50|GuY9d7C1Ox}I4%=gStDHHS;J{X0XOaN&u3^}Pe zhpXk1+=-MCr1I9C8dQ!lCSdDk2s#H8d2p@&@?ntkH`;G3O2`VRi&tma!FqGa<1wvc z<7dZt*T^I&U*zQ``V*knsZ&U88q}J2}3KpRC$O>&n zxM)UUrM*s~m6dV0jeh`u;!rnntBo}p_*$z<1_!4Eku}==hGl7_p_+QkmYmXE>{Q#W zG;j@ zn&4=ROg#I}f=XE4l?bPjmu)N`yi#rWtSBW*Rc9G1A`h-S>R(@)Togt?cC*f65Ir|) z+Qe{oaHu}S)8^|p9`7>Sz(ZZK#FNcSUQa<)RL7$r4~6p|rG0$S#QfI&(I33&FiN~t z4}+POy#tMnoU{@tXm}+PE;fghA}RaqvniA;s#L%`k7L*z-4N~s)%R})G4qYr%EsosCcowL###K4FNpU0-0^7~&Z|=%1(KrZ#BAn_}Esj9|9jQ~x z>5u4n3w2bl_{2i(kfT{x_{6|3ZFueQ8PZ@XB_RGP1?~Q8p|9GU2|x-Sko)C;UoAvrAZVwvq*z@N7Gg23+U=CI_}|=U43PMKZS@)h0UBvF|To=;#)Zb zd3a9kMoQ3wrkRq$(Od)ET~GVuv(_A-1-&a&gc5Gq#*+&vTv$8zu%$^~w#>Bcqk z71NT5cm;&_d)c$-f~CHF6b#>GbX`NVi$W}jND;AoN2xejRF9mMhMju`V-v9W2J(C> z=Spf0X)nAA+f9DZ`yI{k(UCe7oriU^L)@-Js9m_=bl&!zO zkS)8Sz|-hP+C#EAUNy+gK2g-ebM+<8vfT8wtj9XNkJ{)8Dq9T)U1q62`h@)h>4w8n zWUbrlL#t zWqSJ(Fb;haBaQl0VWfniI1P0)Yj1oZYfz;tKOA}^`|gC5lg446xd~SI0tBFGitR5Ex=H*enko_vhU^!1iA4o z-t8oPij7a5UY`B7eaZOn^kTqQG(vKEXIMEyR-XaOs{RX2Y-Nm^<&4(peJ&^YR;R>n zlse&!s~L-?IX)+eYz`rrABbOVlPqNsd?IFkiHo!KR@LqibCUkt?&_1im~0biG;nMh z^muaosD-C|Z{1va-tOIhX-)G9I&91}P*~QwO4G6$mm#$SWs)J+%ZSjsna&^bT#d?H zQrUR;iTaziY`|!|x0ddbY=7Hlyq_u~UQ#O#q)EGNxY^f~^F>reA5`0q)hHq@pl?C{ zgC-*o)o)&k&mTA#c-538Rf|u+9G!}ENJ`~CIb>WvEt}$i`fD&>jRu+bw=`iK`a-VE z6}cSw77l$+5$O{o`e)|wzOQ{HtYWKJ;m*)XHhvHmEWJtTB(YKGfS&8uWMgwiz0VzT zjjXP<&{)ANT_w;=?JWO=oqVA1O@z}@KfuvHAys=>jZTvH2z74i(~@H#yetF zOZDG?8q8wXqHJkypgR;ebjvqvuczK~od#B6?mNF^A6eTuSyidO-})qW@jqWLrffE= z<5vg>R0PSIG+ZEqBM`wNp*U@vm^7&i8$*b+A!6!#vS}%@I<#u;Fiq!Sm~)+y43A={ zSaX$@JS}NER3)kTC=KZFncYms$K@#G`&6+~OR>YxSB>V6ZO@4G&;y9TSipn!(+)hq zWxeY@Ctxz*`sNX2?u+*2)023ZP%PI34M`tmaWS`yeJ{u_Qw7Y8qT;_}sY*>SL|1Ql z^CgYAjSI5P?Ldy@hXq7c;~J28tI82}`EHj_tY)jwH!z>?9LM^l1n$5q{y@cc4aH>a zEj9_Z2&(CeX)e-B7+2na4rv~?HKPOvo+OQwBr{$fo@74y&g?@hUp^sIesY`L`v*TP z^Qj9tWG@i7s%no?e>1*0PrKl5iCO2Zb!{sDPmJUPRlBc3>09H=??fGJ)JY)pJ2S<{ z5F(ZTJ3fffe_$*=GY@UXZ7#WBIc85O02DD&A5$b<>>c?v4in&7W9mUh(4z<9A&e>pY&^&h+2!WoHC^_R!k5FvdI&i=o~w@5WQ_ z4BL1@@C8dxckijT50C>H$U#?SqfVQ~HZ7Vu< zWEJI6h@$Hjz_irWUQ0PC&AMT26QA)uVf|xpxU+gT5U!o%0%_(D66UXArR+Kxb2*^s z<|p!;jH|VfY4rKT&_U;%ai(r#gdRUA(QcZghzH97rz#tjtjce@%*ghN$b)Xz3Bw%P z#D1p7NVGJ2RJH5FV8rrCblCoaIW_Y`rn{tHtY$RW8w&LB^i|YuM?;eFAtXOOUAq}{ z|5z_-PJ$jYMY4md+fAa|4UL2NSr3^1_(I{Xlj{-_akfZyfB6!bV#6P0D8fQ#OX1|^ z@he1j5csXYeetoseK~PV1xWuM&`aoN5JF`L@!w-0uGOWpw9VkRo<7voHli9%QY8kxGYx9lp$M$$6&KRv5olc=J{W2? zdH}3b^}(7+9!hajN?FdH@;aI{zX3mA>hoEXfhNCSn6?1mZn(LW&2fi)G$K9e9|^l{ z32E9AQu?~&abHL%fUFjAExw9`o9n^Bv!}11J@Q!zA>DUIVbEIE!}AQeE{l$sT$FHg z_>elkej}u8ahgvgZl9zoxBWJrWnE1bVQPq~`1`o&tM<$;n*C!wUdC2y*~K_7U&i%4 zZnAQ&*Ua2Y6MSAfs6WusQ6AQ%z!8*K3N8hyRS-SHUoo&S&^!`%HNn(L43gLs?RWYq zvd7w@_l>9lq-3J=krnUyDRu!`4o>dm#xW{WisVr`G-Y;UU_Y-{{R|efw|nU;hB6)14+9<9?sNONTY^ z;-Zajb7XD|IvJW3pXpUp(rWu)b%-OI?sM&9i5HC?*Q1IwMe2_pVjrb?;>MPJNGz}~ zuqJi6mA#m(%)6lzR51Kh5dZL#dHEI7#0%L=>ap|!04U{LkNc{^Ul;#v%V zkJx+3vGwktRaE=_BTu$Sz(PHlF|tVerSdJ^el_PyK4N;!Z>zLnU&fr30-j4gANmA| zvM9SXJ%S=1EgW#Kxs}*|QLH+Df0B$505)M`uW)|;ycS@84A7N45eS!`jNUuB*trP| zRx6++>C3jcs;6`0br7EMWIR`0tiaasw;=U;Y1j z?f>Za36ja!Y5$7;^Ls%+VE(t_4~v7VNdL1bPI44G=3o6kl0SW>zvK14h=72L@cr%D tn`BBB(&R~Y%)bX&XD34>r2VrbL8PgI{KxVmAYlHPrY{i?@@W65{tvPwo^SvF diff --git a/xray/commands/audit/sca/java/gradle.go b/xray/commands/audit/sca/java/gradle.go index d140efffd..933788192 100644 --- a/xray/commands/audit/sca/java/gradle.go +++ b/xray/commands/audit/sca/java/gradle.go @@ -2,11 +2,8 @@ package java import ( _ "embed" - "encoding/base64" - "encoding/json" "errors" "fmt" - "github.com/jfrog/gofrog/datastructures" "os" "os/exec" "path/filepath" @@ -58,60 +55,12 @@ allprojects { var gradleDepTreeJar []byte type depTreeManager struct { - dependenciesTree server *config.ServerDetails releasesRepo string depsRepo string useWrapper bool } -// dependenciesTree represents a map between dependencies to their children dependencies in multiple projects. -type dependenciesTree struct { - tree map[string][]dependenciesPaths -} - -// dependenciesPaths represents a map between dependencies to their children dependencies in a single project. -type dependenciesPaths struct { - Paths map[string]dependenciesPaths `json:"children"` -} - -// The gradle-dep-tree generates a JSON representation for the dependencies for each gradle build file in the project. -// parseDepTreeFiles iterates over those JSONs, and append them to the map of dependencies in dependenciesTree struct. -func (dtp *depTreeManager) parseDepTreeFiles(jsonFiles []byte) error { - outputFiles := strings.Split(strings.TrimSpace(string(jsonFiles)), "\n") - for _, path := range outputFiles { - tree, err := os.ReadFile(strings.TrimSpace(path)) - if err != nil { - return errorutils.CheckError(err) - } - - encodedFileName := path[strings.LastIndex(path, string(os.PathSeparator))+1:] - decodedFileName, err := base64.StdEncoding.DecodeString(encodedFileName) - if err != nil { - return errorutils.CheckError(err) - } - - if err = dtp.appendDependenciesPaths(tree, string(decodedFileName)); err != nil { - return errorutils.CheckError(err) - } - } - return nil -} - -func (dtp *depTreeManager) appendDependenciesPaths(jsonDepTree []byte, fileName string) error { - var deps dependenciesPaths - if err := json.Unmarshal(jsonDepTree, &deps); err != nil { - return errorutils.CheckError(err) - } - if dtp.tree == nil { - dtp.tree = make(map[string][]dependenciesPaths) - } - if len(deps.Paths) > 0 { - dtp.tree[fileName] = append(dtp.tree[fileName], deps) - } - return nil -} - func buildGradleDependencyTree(params *DependencyTreeParams) (dependencyTree []*xrayUtils.GraphNode, uniqueDeps []string, err error) { manager := &depTreeManager{useWrapper: params.UseWrapper} if params.IgnoreConfigFile { @@ -130,7 +79,7 @@ func buildGradleDependencyTree(params *DependencyTreeParams) (dependencyTree []* if err != nil { return } - dependencyTree, uniqueDeps, err = manager.getGraphFromDepTree(outputFileContent) + dependencyTree, uniqueDeps, err = getGraphFromDepTree(outputFileContent) return } @@ -163,7 +112,7 @@ func (dtp *depTreeManager) createDepTreeScriptAndGetDir() (tmpDir string, err er if err != nil { return } - gradleDepTreeJarPath := filepath.Join(tmpDir, string(gradleDepTreeJarFile)) + gradleDepTreeJarPath := filepath.Join(tmpDir, gradleDepTreeJarFile) if err = errorutils.CheckError(os.WriteFile(gradleDepTreeJarPath, gradleDepTreeJar, 0666)); err != nil { return } @@ -237,42 +186,6 @@ func (dtp *depTreeManager) execGradleDepTree(depTreeDir string) (outputFileConte return } -// Assuming we ran gradle-dep-tree, getGraphFromDepTree receives the content of the depTreeOutputFile as input -func (dtp *depTreeManager) getGraphFromDepTree(outputFileContent []byte) ([]*xrayUtils.GraphNode, []string, error) { - if err := dtp.parseDepTreeFiles(outputFileContent); err != nil { - return nil, nil, err - } - var depsGraph []*xrayUtils.GraphNode - uniqueDepsSet := datastructures.MakeSet[string]() - for dependency, children := range dtp.tree { - directDependency := &xrayUtils.GraphNode{ - Id: GavPackageTypeIdentifier + dependency, - Nodes: []*xrayUtils.GraphNode{}, - } - for _, childPath := range children { - populateGradleDependencyTree(directDependency, childPath, uniqueDepsSet) - } - depsGraph = append(depsGraph, directDependency) - } - return depsGraph, uniqueDepsSet.ToSlice(), nil -} - -func populateGradleDependencyTree(currNode *xrayUtils.GraphNode, currNodeChildren dependenciesPaths, uniqueDepsSet *datastructures.Set[string]) { - uniqueDepsSet.Add(currNode.Id) - for gav, children := range currNodeChildren.Paths { - childNode := &xrayUtils.GraphNode{ - Id: GavPackageTypeIdentifier + gav, - Nodes: []*xrayUtils.GraphNode{}, - Parent: currNode, - } - if currNode.NodeHasLoop() { - return - } - populateGradleDependencyTree(childNode, children, uniqueDepsSet) - currNode.Nodes = append(currNode.Nodes, childNode) - } -} - func getDepTreeArtifactoryRepository(remoteRepo string, server *config.ServerDetails) (string, error) { if remoteRepo == "" || server.IsEmpty() { return "", nil diff --git a/xray/commands/audit/sca/java/gradle_test.go b/xray/commands/audit/sca/java/gradle_test.go index e9d09b742..27c4aaaa0 100644 --- a/xray/commands/audit/sca/java/gradle_test.go +++ b/xray/commands/audit/sca/java/gradle_test.go @@ -47,10 +47,10 @@ func TestGradleTreesWithoutConfig(t *testing.T) { // Run getModulesDependencyTrees modulesDependencyTrees, uniqueDeps, err := buildGradleDependencyTree(&DependencyTreeParams{}) if assert.NoError(t, err) && assert.NotNil(t, modulesDependencyTrees) { - assert.Len(t, uniqueDeps, 11) - assert.Len(t, modulesDependencyTrees, 2) + assert.Len(t, uniqueDeps, 9) + assert.Len(t, modulesDependencyTrees, 5) // Check module - module := sca.GetAndAssertNode(t, modulesDependencyTrees, "webservice") + module := sca.GetAndAssertNode(t, modulesDependencyTrees, "org.jfrog.example.gradle:webservice:1.0") assert.Len(t, module.Nodes, 7) // Check direct dependency @@ -71,10 +71,10 @@ func TestGradleTreesWithConfig(t *testing.T) { // Run getModulesDependencyTrees modulesDependencyTrees, uniqueDeps, err := buildGradleDependencyTree(&DependencyTreeParams{UseWrapper: true}) if assert.NoError(t, err) && assert.NotNil(t, modulesDependencyTrees) { - assert.Len(t, modulesDependencyTrees, 3) - assert.Len(t, uniqueDeps, 11) + assert.Len(t, modulesDependencyTrees, 5) + assert.Len(t, uniqueDeps, 8) // Check module - module := sca.GetAndAssertNode(t, modulesDependencyTrees, "api") + module := sca.GetAndAssertNode(t, modulesDependencyTrees, "org.jfrog.test.gradle.publish:api:1.0-SNAPSHOT") assert.Len(t, module.Nodes, 4) // Check direct dependency @@ -86,22 +86,6 @@ func TestGradleTreesWithConfig(t *testing.T) { } } -func TestGradleTreesExcludeTestDeps(t *testing.T) { - // Create and change directory to test workspace - tempDirPath, cleanUp := sca.CreateTestWorkspace(t, "gradle-example-ci-server") - defer cleanUp() - assert.NoError(t, os.Chmod(filepath.Join(tempDirPath, "gradlew"), 0700)) - - // Run getModulesDependencyTrees - modulesDependencyTrees, uniqueDeps, err := buildGradleDependencyTree(&DependencyTreeParams{UseWrapper: true}) - if assert.NoError(t, err) && assert.NotNil(t, modulesDependencyTrees) { - assert.Len(t, modulesDependencyTrees, 2) - assert.Len(t, uniqueDeps, 11) - // Check direct dependency - assert.Nil(t, sca.GetModule(modulesDependencyTrees, "services")) - } -} - func TestIsGradleWrapperExist(t *testing.T) { // Check Gradle wrapper doesn't exist isWrapperExist, err := isGradleWrapperExist() @@ -168,67 +152,6 @@ func TestGetDepTreeArtifactoryRepository(t *testing.T) { } } -func TestGetGraphFromDepTree(t *testing.T) { - // Create and change directory to test workspace - tempDirPath, cleanUp := sca.CreateTestWorkspace(t, "gradle-example-ci-server") - defer func() { - cleanUp() - }() - assert.NoError(t, os.Chmod(filepath.Join(tempDirPath, "gradlew"), 0700)) - testCase := struct { - name string - expectedTree map[string]map[string]string - expectedUniqueDeps []string - }{ - name: "ValidOutputFileContent", - expectedTree: map[string]map[string]string{ - GavPackageTypeIdentifier + "shared": {}, - GavPackageTypeIdentifier + filepath.Base(tempDirPath): {}, - GavPackageTypeIdentifier + "services": {}, - GavPackageTypeIdentifier + "webservice": { - GavPackageTypeIdentifier + "junit:junit:4.11": "", - GavPackageTypeIdentifier + "commons-io:commons-io:1.2": "", - GavPackageTypeIdentifier + "org.apache.wicket:wicket:1.3.7": "", - GavPackageTypeIdentifier + "org.jfrog.example.gradle:shared:1.0": "", - GavPackageTypeIdentifier + "org.jfrog.example.gradle:api:1.0": "", - GavPackageTypeIdentifier + "commons-lang:commons-lang:2.4": "", - GavPackageTypeIdentifier + "commons-collections:commons-collections:3.2": "", - }, - GavPackageTypeIdentifier + "api": { - GavPackageTypeIdentifier + "org.apache.wicket:wicket:1.3.7": "", - GavPackageTypeIdentifier + "org.jfrog.example.gradle:shared:1.0": "", - GavPackageTypeIdentifier + "commons-lang:commons-lang:2.4": "", - }, - }, - expectedUniqueDeps: []string{ - GavPackageTypeIdentifier + "webservice", - GavPackageTypeIdentifier + "junit:junit:4.11", - GavPackageTypeIdentifier + "commons-io:commons-io:1.2", - GavPackageTypeIdentifier + "org.apache.wicket:wicket:1.3.7", - GavPackageTypeIdentifier + "org.jfrog.example.gradle:shared:1.0", - GavPackageTypeIdentifier + "org.jfrog.example.gradle:api:1.0", - GavPackageTypeIdentifier + "commons-collections:commons-collections:3.2", - GavPackageTypeIdentifier + "api", - GavPackageTypeIdentifier + "commons-lang:commons-lang:2.4", - GavPackageTypeIdentifier + "org.hamcrest:hamcrest-core:1.3", - GavPackageTypeIdentifier + "org.slf4j:slf4j-api:1.4.2", - }, - } - - manager := &depTreeManager{} - outputFileContent, err := manager.runGradleDepTree() - assert.NoError(t, err) - depTree, uniqueDeps, err := (&depTreeManager{}).getGraphFromDepTree(outputFileContent) - assert.NoError(t, err) - assert.ElementsMatch(t, uniqueDeps, testCase.expectedUniqueDeps, "First is actual, Second is Expected") - - for _, dependency := range depTree { - depChild, exists := testCase.expectedTree[dependency.Id] - assert.True(t, exists) - assert.Equal(t, len(depChild), len(dependency.Nodes)) - } -} - func TestCreateDepTreeScript(t *testing.T) { manager := &depTreeManager{} tmpDir, err := manager.createDepTreeScriptAndGetDir() diff --git a/xray/commands/audit/sca/java/javautils.go b/xray/commands/audit/sca/java/javautils.go index 9653de43f..37009cb5b 100644 --- a/xray/commands/audit/sca/java/javautils.go +++ b/xray/commands/audit/sca/java/javautils.go @@ -1,12 +1,15 @@ package java import ( + "encoding/json" "github.com/jfrog/gofrog/datastructures" "github.com/jfrog/jfrog-cli-core/v2/utils/config" "github.com/jfrog/jfrog-cli-core/v2/utils/coreutils" xrayutils "github.com/jfrog/jfrog-cli-core/v2/xray/utils" xrayUtils "github.com/jfrog/jfrog-client-go/xray/services/utils" + "os" "strconv" + "strings" "time" buildinfo "github.com/jfrog/build-info-go/entities" @@ -58,7 +61,7 @@ func createGavDependencyTree(buildConfig *artifactoryUtils.BuildConfiguration) ( if len(generatedBuildsInfos) == 0 { return nil, nil, errorutils.CheckErrorf("Couldn't find build " + buildName + "/" + buildNumber) } - modules := []*xrayUtils.GraphNode{} + var modules []*xrayUtils.GraphNode uniqueDepsSet := datastructures.MakeSet[string]() for _, module := range generatedBuildsInfos[0].Modules { modules = append(modules, addModuleTree(module, uniqueDepsSet)) @@ -173,3 +176,73 @@ func (dm *dependencyMultimap) putChild(parent string, child *buildinfo.Dependenc func (dm *dependencyMultimap) getChildren(parent string) map[string]*buildinfo.Dependency { return dm.multimap[parent] } + +// The structure of a dependency tree of a module in a Gradle/Maven project, as created by the gradle-dep-tree and maven-dep-tree plugins. +type moduleDepTree struct { + Root string `json:"root"` + Nodes map[string]depTreeNode `json:"nodes"` +} + +type depTreeNode struct { + Children []string `json:"children"` +} + +// getGraphFromDepTree reads the output files of the gradle-dep-tree and maven-dep-tree plugins and returns them as a slice of GraphNodes. +// It takes the output of the plugin's run (which is a byte representation of a list of paths of the output files, separated by newlines) as input. +func getGraphFromDepTree(depTreeOutput []byte) (depsGraph []*xrayUtils.GraphNode, uniqueDeps []string, err error) { + modules, err := parseDepTreeFiles(depTreeOutput) + if err != nil { + return + } + uniqueDepsSet := datastructures.MakeSet[string]() + for _, moduleTree := range modules { + directDependency := &xrayUtils.GraphNode{ + Id: GavPackageTypeIdentifier + moduleTree.Root, + Nodes: []*xrayUtils.GraphNode{}, + } + populateDependencyTree(directDependency, moduleTree.Root, moduleTree, uniqueDepsSet) + depsGraph = append(depsGraph, directDependency) + } + uniqueDeps = uniqueDepsSet.ToSlice() + return +} + +func populateDependencyTree(currNode *xrayUtils.GraphNode, currNodeId string, moduleTree *moduleDepTree, uniqueDepsSet *datastructures.Set[string]) { + if currNode.NodeHasLoop() { + return + } + for _, childId := range moduleTree.Nodes[currNodeId].Children { + childGav := GavPackageTypeIdentifier + childId + childNode := &xrayUtils.GraphNode{ + Id: childGav, + Nodes: []*xrayUtils.GraphNode{}, + Parent: currNode, + } + uniqueDepsSet.Add(childGav) + populateDependencyTree(childNode, childId, moduleTree, uniqueDepsSet) + currNode.Nodes = append(currNode.Nodes, childNode) + } +} + +func parseDepTreeFiles(jsonFilePaths []byte) ([]*moduleDepTree, error) { + outputFilePaths := strings.Split(strings.TrimSpace(string(jsonFilePaths)), "\n") + var modules []*moduleDepTree + for _, path := range outputFilePaths { + results, err := parseDepTreeFile(path) + if err != nil { + return nil, err + } + modules = append(modules, results) + } + return modules, nil +} + +func parseDepTreeFile(path string) (results *moduleDepTree, err error) { + depTreeJson, err := os.ReadFile(strings.TrimSpace(path)) + if errorutils.CheckError(err) != nil { + return + } + results = &moduleDepTree{} + err = errorutils.CheckError(json.Unmarshal(depTreeJson, &results)) + return +} diff --git a/xray/commands/audit/sca/java/javautils_test.go b/xray/commands/audit/sca/java/javautils_test.go new file mode 100644 index 000000000..325a59408 --- /dev/null +++ b/xray/commands/audit/sca/java/javautils_test.go @@ -0,0 +1,68 @@ +package java + +import ( + "github.com/jfrog/jfrog-cli-core/v2/xray/commands/audit/sca" + "github.com/stretchr/testify/assert" + "os" + "path/filepath" + "testing" +) + +func TestGetGraphFromDepTree(t *testing.T) { + // Create and change directory to test workspace + tempDirPath, cleanUp := sca.CreateTestWorkspace(t, "gradle-example-ci-server") + defer func() { + cleanUp() + }() + assert.NoError(t, os.Chmod(filepath.Join(tempDirPath, "gradlew"), 0700)) + testCase := struct { + name string + expectedTree map[string]map[string]string + expectedUniqueDeps []string + }{ + name: "ValidOutputFileContent", + expectedTree: map[string]map[string]string{ + GavPackageTypeIdentifier + "org.jfrog.example.gradle:shared:1.0": {}, + GavPackageTypeIdentifier + "org.jfrog.example.gradle:" + filepath.Base(tempDirPath) + ":1.0": {}, + GavPackageTypeIdentifier + "org.jfrog.example.gradle:services:1.0": {}, + GavPackageTypeIdentifier + "org.jfrog.example.gradle:webservice:1.0": { + GavPackageTypeIdentifier + "junit:junit:4.11": "", + GavPackageTypeIdentifier + "commons-io:commons-io:1.2": "", + GavPackageTypeIdentifier + "org.apache.wicket:wicket:1.3.7": "", + GavPackageTypeIdentifier + "org.jfrog.example.gradle:shared:1.0": "", + GavPackageTypeIdentifier + "org.jfrog.example.gradle:api:1.0": "", + GavPackageTypeIdentifier + "commons-lang:commons-lang:2.4": "", + GavPackageTypeIdentifier + "commons-collections:commons-collections:3.2": "", + }, + GavPackageTypeIdentifier + "org.jfrog.example.gradle:api:1.0": { + GavPackageTypeIdentifier + "org.apache.wicket:wicket:1.3.7": "", + GavPackageTypeIdentifier + "org.jfrog.example.gradle:shared:1.0": "", + GavPackageTypeIdentifier + "commons-lang:commons-lang:2.4": "", + }, + }, + expectedUniqueDeps: []string{ + GavPackageTypeIdentifier + "junit:junit:4.11", + GavPackageTypeIdentifier + "commons-io:commons-io:1.2", + GavPackageTypeIdentifier + "org.apache.wicket:wicket:1.3.7", + GavPackageTypeIdentifier + "org.jfrog.example.gradle:shared:1.0", + GavPackageTypeIdentifier + "org.jfrog.example.gradle:api:1.0", + GavPackageTypeIdentifier + "commons-collections:commons-collections:3.2", + GavPackageTypeIdentifier + "commons-lang:commons-lang:2.4", + GavPackageTypeIdentifier + "org.hamcrest:hamcrest-core:1.3", + GavPackageTypeIdentifier + "org.slf4j:slf4j-api:1.4.2", + }, + } + + manager := &depTreeManager{} + outputFileContent, err := manager.runGradleDepTree() + assert.NoError(t, err) + depTree, uniqueDeps, err := getGraphFromDepTree(outputFileContent) + assert.NoError(t, err) + assert.ElementsMatch(t, uniqueDeps, testCase.expectedUniqueDeps, "First is actual, Second is Expected") + + for _, dependency := range depTree { + depChild, exists := testCase.expectedTree[dependency.Id] + assert.True(t, exists) + assert.Equal(t, len(depChild), len(dependency.Nodes)) + } +} From 29d9856225d2e40c64d1ce3d94bb7169c6875361 Mon Sep 17 00:00:00 2001 From: Robi Nino Date: Mon, 2 Oct 2023 15:59:48 +0300 Subject: [PATCH 10/16] Transfer - Additional logs (#974) --- .../transferfiles/delayedartifactshandler.go | 1 + .../commands/transferfiles/errorshandler.go | 16 ++++++++++++++-- artifactory/commands/transferfiles/manager.go | 5 +++++ 3 files changed, 20 insertions(+), 2 deletions(-) diff --git a/artifactory/commands/transferfiles/delayedartifactshandler.go b/artifactory/commands/transferfiles/delayedartifactshandler.go index adab43f5c..11af1d3cb 100644 --- a/artifactory/commands/transferfiles/delayedartifactshandler.go +++ b/artifactory/commands/transferfiles/delayedartifactshandler.go @@ -161,6 +161,7 @@ func handleDelayedArtifactsFiles(filesToConsume []string, base phaseBase, delayU } func consumeDelayedArtifactsFiles(pcWrapper *producerConsumerWrapper, filesToConsume []string, uploadChunkChan chan UploadedChunk, base phaseBase, delayHelper delayUploadHelper, errorsChannelMng *ErrorsChannelMng) error { + log.Debug(fmt.Sprintf("Starting to handle delayed artifacts files. Found %d files.", len(filesToConsume))) for _, filePath := range filesToConsume { log.Debug("Handling delayed artifacts file: '" + filePath + "'") delayedArtifactsFile, err := readDelayFile(filePath) diff --git a/artifactory/commands/transferfiles/errorshandler.go b/artifactory/commands/transferfiles/errorshandler.go index f67079039..db9a8cb3e 100644 --- a/artifactory/commands/transferfiles/errorshandler.go +++ b/artifactory/commands/transferfiles/errorshandler.go @@ -11,6 +11,7 @@ import ( "github.com/jfrog/jfrog-client-go/utils/io/fileutils" "github.com/jfrog/jfrog-client-go/utils/log" "os" + "path" "time" ) @@ -189,7 +190,7 @@ func (mng *TransferErrorsMng) writeErrorContent(e ExtendedFileUploadStatusRespon } func (mng *TransferErrorsMng) writeSkippedErrorContent(e ExtendedFileUploadStatusResponse) error { - log.Debug(fmt.Sprintf("Writing '%s' to file %s", e.Reason, mng.errorWriterMng.skipped.filePath)) + logWritingArtifact(e, mng.errorWriterMng.skipped.filePath) mng.errorWriterMng.skipped.writer.Write(e) mng.errorWriterMng.skipped.errorCount++ // If file contains maximum number of errors - create and write to a new errors file @@ -213,7 +214,7 @@ func (mng *TransferErrorsMng) writeSkippedErrorContent(e ExtendedFileUploadStatu } func (mng *TransferErrorsMng) writeRetryableErrorContent(e ExtendedFileUploadStatusResponse) error { - log.Debug(fmt.Sprintf("Writing '%s' to file %s", e.Reason, mng.errorWriterMng.retryable.filePath)) + logWritingArtifact(e, mng.errorWriterMng.retryable.filePath) mng.errorWriterMng.retryable.writer.Write(e) mng.errorWriterMng.retryable.errorCount++ // If file contains maximum number of errors - create and write to a new errors file @@ -236,6 +237,17 @@ func (mng *TransferErrorsMng) writeRetryableErrorContent(e ExtendedFileUploadSta return nil } +func logWritingArtifact(e ExtendedFileUploadStatusResponse, errorsFilePath string) { + if log.GetLogger().GetLogLevel() != log.DEBUG { + return + } + msg := fmt.Sprintf("Writing artifact '%s' to errors file '%s'.", path.Join(e.Repo, e.Path, e.Name), errorsFilePath) + if e.Reason != "" { + msg += fmt.Sprintf(" Reason: '%s'.", e.Reason) + } + log.Debug(msg) +} + func (writerMng *errorWriter) closeWriter() error { // Close content writer and move output file to our working directory if writerMng.writer == nil { diff --git a/artifactory/commands/transferfiles/manager.go b/artifactory/commands/transferfiles/manager.go index 2b825f88b..e9904b592 100644 --- a/artifactory/commands/transferfiles/manager.go +++ b/artifactory/commands/transferfiles/manager.go @@ -228,6 +228,8 @@ func runProducerConsumers(pcWrapper *producerConsumerWrapper) (executionErr erro go func() { // Wait till notified that the builder has no additional tasks, and close the builder producer consumer. <-pcWrapper.chunkBuilderProducerConsumer.GetFinishedNotification() + log.Debug("Chunk builder producer consumer has completed all tasks. " + + "All files relevant to this phase were found and added to chunks that are being uploaded...") pcWrapper.chunkBuilderProducerConsumer.Done() }() @@ -261,6 +263,7 @@ func pollUploads(phaseBase *phaseBase, srcUpService *srcUserPluginService, uploa } for i := 0; ; i++ { if ShouldStop(phaseBase, nil, errorsChannelMng) { + log.Debug("Stop signal received while polling on uploads...") return } time.Sleep(waitTimeBetweenChunkStatusSeconds * time.Second) @@ -284,6 +287,7 @@ func pollUploads(phaseBase *phaseBase, srcUpService *srcUserPluginService, uploa // it will be written to the error channel if chunksLifeCycleManager.totalChunks == 0 { if shouldStopPolling(doneChan) { + log.Debug("Stopping to poll on uploads...") return } continue @@ -386,6 +390,7 @@ func handleChunksStatuses(phase *phaseBase, chunksStatus *api.UploadChunksStatus stopped := handleFilesOfCompletedChunk(chunk.Files, errorsChannelMng) // In case an error occurred while writing errors status's to the errors file - stop transferring. if stopped { + log.Debug("Stop signal received while handling chunks statuses...") return true } err = setChunkCompletedInRepoSnapshot(phase.stateManager, chunk.Files) From 81959c18ab18778109de00b407b30b0a092a206f Mon Sep 17 00:00:00 2001 From: Robi Nino Date: Mon, 2 Oct 2023 16:11:18 +0300 Subject: [PATCH 11/16] Transfer - Fix handling delayed files condition (#973) --- artifactory/commands/transferfiles/delayedartifactshandler.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/artifactory/commands/transferfiles/delayedartifactshandler.go b/artifactory/commands/transferfiles/delayedartifactshandler.go index 11af1d3cb..31219f00e 100644 --- a/artifactory/commands/transferfiles/delayedartifactshandler.go +++ b/artifactory/commands/transferfiles/delayedartifactshandler.go @@ -152,7 +152,7 @@ func handleDelayedArtifactsFiles(filesToConsume []string, base phaseBase, delayU delayAction := func(pBase phaseBase, addedDelayFiles []string) error { // We call this method as a recursion in order to have inner order base on the comparison function list. // Remove the first delay comparison function one by one to no longer delay it until the list is empty. - if len(filesToConsume) > 0 && len(delayUploadComparisonFunctions) > 0 { + if len(addedDelayFiles) > 0 && len(delayUploadComparisonFunctions) > 0 { return handleDelayedArtifactsFiles(addedDelayFiles, pBase, delayUploadComparisonFunctions[1:]) } return nil From c17c0f5f731ee23536d616fd4d460155390cbbc9 Mon Sep 17 00:00:00 2001 From: Assaf Attias <49212512+attiasas@users.noreply.github.com> Date: Tue, 3 Oct 2023 11:35:35 +0300 Subject: [PATCH 12/16] Fix Sarif output driver issues and Xray Sca locations (#968) --- utils/coreutils/techutils.go | 24 +++-- .../jas/applicability/applicabilitymanager.go | 7 +- .../applicabilitymanager_test.go | 6 +- xray/commands/audit/jas/common.go | 22 ++++- xray/commands/audit/jas/commons_test.go | 2 +- xray/commands/audit/jas/iac/iacscanner.go | 7 +- .../commands/audit/jas/iac/iacscanner_test.go | 4 +- xray/commands/audit/jas/sast/sastscanner.go | 7 +- .../audit/jas/sast/sastscanner_test.go | 4 +- .../audit/jas/secrets/secretsscanner.go | 7 +- .../audit/jas/secrets/secretsscanner_test.go | 4 +- xray/commands/audit/jasrunner.go | 3 - xray/utils/analyzermanager.go | 6 -- xray/utils/resultwriter.go | 91 +++++++++++++------ xray/utils/resultwriter_test.go | 59 ++++++++++++ xray/utils/sarifutils.go | 71 ++------------- xray/utils/sarifutils_test.go | 27 ++++++ xray/utils/test_sarifutils.go | 64 +++++++++++++ 18 files changed, 279 insertions(+), 136 deletions(-) create mode 100644 xray/utils/test_sarifutils.go diff --git a/utils/coreutils/techutils.go b/utils/coreutils/techutils.go index 58ea37a44..3880da6ad 100644 --- a/utils/coreutils/techutils.go +++ b/utils/coreutils/techutils.go @@ -44,8 +44,8 @@ type TechData struct { ciSetupSupport bool // Whether Contextual Analysis supported in this technology. applicabilityScannable bool - // The file that handles the project's dependencies. - packageDescriptor string + // The files that handle the project's dependencies. + packageDescriptors []string // Formal name of the technology formal string // The executable name of the technology @@ -60,21 +60,21 @@ var technologiesData = map[Technology]TechData{ Maven: { indicators: []string{"pom.xml"}, ciSetupSupport: true, - packageDescriptor: "pom.xml", + packageDescriptors: []string{"pom.xml"}, execCommand: "mvn", applicabilityScannable: true, }, Gradle: { indicators: []string{".gradle", ".gradle.kts"}, ciSetupSupport: true, - packageDescriptor: "build.gradle, build.gradle.kts", + packageDescriptors: []string{"build.gradle", "build.gradle.kts"}, applicabilityScannable: true, }, Npm: { indicators: []string{"package.json", "package-lock.json", "npm-shrinkwrap.json"}, exclude: []string{".yarnrc.yml", "yarn.lock", ".yarn"}, ciSetupSupport: true, - packageDescriptor: "package.json", + packageDescriptors: []string{"package.json"}, formal: string(Npm), packageVersionOperator: "@", packageInstallationCommand: "install", @@ -82,26 +82,27 @@ var technologiesData = map[Technology]TechData{ }, Yarn: { indicators: []string{".yarnrc.yml", "yarn.lock", ".yarn"}, - packageDescriptor: "package.json", + packageDescriptors: []string{"package.json"}, packageVersionOperator: "@", applicabilityScannable: true, }, Go: { indicators: []string{"go.mod"}, - packageDescriptor: "go.mod", + packageDescriptors: []string{"go.mod"}, packageVersionOperator: "@v", packageInstallationCommand: "get", }, Pip: { packageType: Pypi, indicators: []string{"setup.py", "requirements.txt"}, + packageDescriptors: []string{"setup.py", "requirements.txt"}, exclude: []string{"Pipfile", "Pipfile.lock", "pyproject.toml", "poetry.lock"}, applicabilityScannable: true, }, Pipenv: { packageType: Pypi, indicators: []string{"Pipfile", "Pipfile.lock"}, - packageDescriptor: "Pipfile", + packageDescriptors: []string{"Pipfile"}, packageVersionOperator: "==", packageInstallationCommand: "install", applicabilityScannable: true, @@ -153,11 +154,8 @@ func (tech Technology) GetPackageType() string { return technologiesData[tech].packageType } -func (tech Technology) GetPackageDescriptor() string { - if technologiesData[tech].packageDescriptor == "" { - return tech.ToFormal() + " Package Descriptor" - } - return technologiesData[tech].packageDescriptor +func (tech Technology) GetPackageDescriptor() []string { + return technologiesData[tech].packageDescriptors } func (tech Technology) IsCiSetup() bool { diff --git a/xray/commands/audit/jas/applicability/applicabilitymanager.go b/xray/commands/audit/jas/applicability/applicabilitymanager.go index c34318c00..202e3d9ff 100644 --- a/xray/commands/audit/jas/applicability/applicabilitymanager.go +++ b/xray/commands/audit/jas/applicability/applicabilitymanager.go @@ -17,8 +17,9 @@ import ( ) const ( - applicabilityScanType = "analyze-applicability" - applicabilityScanCommand = "ca" + applicabilityScanType = "analyze-applicability" + applicabilityScanCommand = "ca" + applicabilityDocsUrlSuffix = "contextual-analysis" ) type ApplicabilityScanManager struct { @@ -116,7 +117,7 @@ func (asm *ApplicabilityScanManager) Run(module jfrogappsconfig.Module) (err err if err = asm.runAnalyzerManager(); err != nil { return } - workingDirResults, err := jas.ReadJasScanRunsFromFile(asm.scanner.ResultsFileName, module.SourceRoot) + workingDirResults, err := jas.ReadJasScanRunsFromFile(asm.scanner.ResultsFileName, module.SourceRoot, applicabilityDocsUrlSuffix) if err != nil { return } diff --git a/xray/commands/audit/jas/applicability/applicabilitymanager_test.go b/xray/commands/audit/jas/applicability/applicabilitymanager_test.go index 98d54d28b..554909181 100644 --- a/xray/commands/audit/jas/applicability/applicabilitymanager_test.go +++ b/xray/commands/audit/jas/applicability/applicabilitymanager_test.go @@ -282,7 +282,7 @@ func TestParseResults_EmptyResults_AllCvesShouldGetUnknown(t *testing.T) { // Act var err error - applicabilityManager.applicabilityScanResults, err = jas.ReadJasScanRunsFromFile(applicabilityManager.scanner.ResultsFileName, scanner.JFrogAppsConfig.Modules[0].SourceRoot) + applicabilityManager.applicabilityScanResults, err = jas.ReadJasScanRunsFromFile(applicabilityManager.scanner.ResultsFileName, scanner.JFrogAppsConfig.Modules[0].SourceRoot, applicabilityDocsUrlSuffix) if assert.NoError(t, err) { assert.Len(t, applicabilityManager.applicabilityScanResults, 1) @@ -299,7 +299,7 @@ func TestParseResults_ApplicableCveExist(t *testing.T) { // Act var err error - applicabilityManager.applicabilityScanResults, err = jas.ReadJasScanRunsFromFile(applicabilityManager.scanner.ResultsFileName, scanner.JFrogAppsConfig.Modules[0].SourceRoot) + applicabilityManager.applicabilityScanResults, err = jas.ReadJasScanRunsFromFile(applicabilityManager.scanner.ResultsFileName, scanner.JFrogAppsConfig.Modules[0].SourceRoot, applicabilityDocsUrlSuffix) if assert.NoError(t, err) && assert.NotNil(t, applicabilityManager.applicabilityScanResults) { assert.Len(t, applicabilityManager.applicabilityScanResults, 1) @@ -316,7 +316,7 @@ func TestParseResults_AllCvesNotApplicable(t *testing.T) { // Act var err error - applicabilityManager.applicabilityScanResults, err = jas.ReadJasScanRunsFromFile(applicabilityManager.scanner.ResultsFileName, scanner.JFrogAppsConfig.Modules[0].SourceRoot) + applicabilityManager.applicabilityScanResults, err = jas.ReadJasScanRunsFromFile(applicabilityManager.scanner.ResultsFileName, scanner.JFrogAppsConfig.Modules[0].SourceRoot, applicabilityDocsUrlSuffix) if assert.NoError(t, err) && assert.NotNil(t, applicabilityManager.applicabilityScanResults) { assert.Len(t, applicabilityManager.applicabilityScanResults, 1) diff --git a/xray/commands/audit/jas/common.go b/xray/commands/audit/jas/common.go index fa7d508ef..be165aa32 100644 --- a/xray/commands/audit/jas/common.go +++ b/xray/commands/audit/jas/common.go @@ -7,6 +7,7 @@ import ( "path/filepath" "strings" "testing" + "unicode" jfrogappsconfig "github.com/jfrog/jfrog-apps-config/go" rtutils "github.com/jfrog/jfrog-cli-core/v2/artifactory/utils" @@ -127,7 +128,7 @@ func deleteJasProcessFiles(configFile string, resultFile string) error { return errorutils.CheckError(err) } -func ReadJasScanRunsFromFile(fileName, wd string) (sarifRuns []*sarif.Run, err error) { +func ReadJasScanRunsFromFile(fileName, wd, informationUrlSuffix string) (sarifRuns []*sarif.Run, err error) { if sarifRuns, err = utils.ReadScanRunsFromFile(fileName); err != nil { return } @@ -137,12 +138,31 @@ func ReadJasScanRunsFromFile(fileName, wd string) (sarifRuns []*sarif.Run, err e // Also used to calculate relative paths if needed with it sarifRun.Invocations[0].WorkingDirectory.WithUri(wd) // Process runs values + fillMissingRequiredDriverInformation(utils.BaseDocumentationURL+informationUrlSuffix, utils.GetAnalyzerManagerVersion(), sarifRun) sarifRun.Results = excludeSuppressResults(sarifRun.Results) addScoreToRunRules(sarifRun) } return } +func fillMissingRequiredDriverInformation(defaultJasInformationUri, defaultVersion string, run *sarif.Run) { + driver := run.Tool.Driver + if driver.InformationURI == nil { + driver.InformationURI = &defaultJasInformationUri + } + if driver.Version == nil || !isValidVersion(*driver.Version) { + driver.Version = &defaultVersion + } +} + +func isValidVersion(version string) bool { + if len(version) == 0 { + return false + } + firstChar := rune(version[0]) + return unicode.IsDigit(firstChar) +} + func excludeSuppressResults(sarifResults []*sarif.Result) []*sarif.Result { results := []*sarif.Result{} for _, sarifResult := range sarifResults { diff --git a/xray/commands/audit/jas/commons_test.go b/xray/commands/audit/jas/commons_test.go index d600d7a73..c43cf94ab 100644 --- a/xray/commands/audit/jas/commons_test.go +++ b/xray/commands/audit/jas/commons_test.go @@ -126,4 +126,4 @@ func TestGetExcludePatterns(t *testing.T) { assert.ElementsMatch(t, actualExcludePatterns, expectedExcludePatterns) }) } -} \ No newline at end of file +} diff --git a/xray/commands/audit/jas/iac/iacscanner.go b/xray/commands/audit/jas/iac/iacscanner.go index f2ea4984b..402557e3e 100644 --- a/xray/commands/audit/jas/iac/iacscanner.go +++ b/xray/commands/audit/jas/iac/iacscanner.go @@ -12,8 +12,9 @@ import ( ) const ( - iacScannerType = "iac-scan-modules" - iacScanCommand = "iac" + iacScannerType = "iac-scan-modules" + iacScanCommand = "iac" + iacDocsUrlSuffix = "infrastructure-as-code-iac" ) type IacScanManager struct { @@ -60,7 +61,7 @@ func (iac *IacScanManager) Run(module jfrogappsconfig.Module) (err error) { if err = iac.runAnalyzerManager(); err != nil { return } - workingDirResults, err := jas.ReadJasScanRunsFromFile(iac.scanner.ResultsFileName, module.SourceRoot) + workingDirResults, err := jas.ReadJasScanRunsFromFile(iac.scanner.ResultsFileName, module.SourceRoot, iacDocsUrlSuffix) if err != nil { return } diff --git a/xray/commands/audit/jas/iac/iacscanner_test.go b/xray/commands/audit/jas/iac/iacscanner_test.go index 595715bfd..1f9482ded 100644 --- a/xray/commands/audit/jas/iac/iacscanner_test.go +++ b/xray/commands/audit/jas/iac/iacscanner_test.go @@ -59,7 +59,7 @@ func TestIacParseResults_EmptyResults(t *testing.T) { // Act var err error - iacScanManager.iacScannerResults, err = jas.ReadJasScanRunsFromFile(iacScanManager.scanner.ResultsFileName, scanner.JFrogAppsConfig.Modules[0].SourceRoot) + iacScanManager.iacScannerResults, err = jas.ReadJasScanRunsFromFile(iacScanManager.scanner.ResultsFileName, scanner.JFrogAppsConfig.Modules[0].SourceRoot, iacDocsUrlSuffix) if assert.NoError(t, err) && assert.NotNil(t, iacScanManager.iacScannerResults) { assert.Len(t, iacScanManager.iacScannerResults, 1) assert.Empty(t, iacScanManager.iacScannerResults[0].Results) @@ -75,7 +75,7 @@ func TestIacParseResults_ResultsContainIacViolations(t *testing.T) { // Act var err error - iacScanManager.iacScannerResults, err = jas.ReadJasScanRunsFromFile(iacScanManager.scanner.ResultsFileName, scanner.JFrogAppsConfig.Modules[0].SourceRoot) + iacScanManager.iacScannerResults, err = jas.ReadJasScanRunsFromFile(iacScanManager.scanner.ResultsFileName, scanner.JFrogAppsConfig.Modules[0].SourceRoot, iacDocsUrlSuffix) if assert.NoError(t, err) && assert.NotNil(t, iacScanManager.iacScannerResults) { assert.Len(t, iacScanManager.iacScannerResults, 1) assert.Len(t, iacScanManager.iacScannerResults[0].Results, 4) diff --git a/xray/commands/audit/jas/sast/sastscanner.go b/xray/commands/audit/jas/sast/sastscanner.go index 2052a5361..578bc6775 100644 --- a/xray/commands/audit/jas/sast/sastscanner.go +++ b/xray/commands/audit/jas/sast/sastscanner.go @@ -14,8 +14,9 @@ import ( ) const ( - sastScannerType = "sast" - sastScanCommand = "zd" + sastScannerType = "sast" + sastScanCommand = "zd" + sastDocsUrlSuffix = "sast" ) type SastScanManager struct { @@ -55,7 +56,7 @@ func (ssm *SastScanManager) Run(module jfrogappsconfig.Module) (err error) { if err = ssm.runAnalyzerManager(filepath.Dir(ssm.scanner.AnalyzerManager.AnalyzerManagerFullPath)); err != nil { return } - workingDirRuns, err := jas.ReadJasScanRunsFromFile(scanner.ResultsFileName, module.SourceRoot) + workingDirRuns, err := jas.ReadJasScanRunsFromFile(scanner.ResultsFileName, module.SourceRoot, sastDocsUrlSuffix) if err != nil { return } diff --git a/xray/commands/audit/jas/sast/sastscanner_test.go b/xray/commands/audit/jas/sast/sastscanner_test.go index 036c028c8..43eedf705 100644 --- a/xray/commands/audit/jas/sast/sastscanner_test.go +++ b/xray/commands/audit/jas/sast/sastscanner_test.go @@ -36,7 +36,7 @@ func TestSastParseResults_EmptyResults(t *testing.T) { // Act var err error - sastScanManager.sastScannerResults, err = jas.ReadJasScanRunsFromFile(sastScanManager.scanner.ResultsFileName, scanner.JFrogAppsConfig.Modules[0].SourceRoot) + sastScanManager.sastScannerResults, err = jas.ReadJasScanRunsFromFile(sastScanManager.scanner.ResultsFileName, scanner.JFrogAppsConfig.Modules[0].SourceRoot, sastDocsUrlSuffix) // Assert if assert.NoError(t, err) && assert.NotNil(t, sastScanManager.sastScannerResults) { @@ -57,7 +57,7 @@ func TestSastParseResults_ResultsContainIacViolations(t *testing.T) { // Act var err error - sastScanManager.sastScannerResults, err = jas.ReadJasScanRunsFromFile(sastScanManager.scanner.ResultsFileName, scanner.JFrogAppsConfig.Modules[0].SourceRoot) + sastScanManager.sastScannerResults, err = jas.ReadJasScanRunsFromFile(sastScanManager.scanner.ResultsFileName, scanner.JFrogAppsConfig.Modules[0].SourceRoot, sastDocsUrlSuffix) // Assert if assert.NoError(t, err) && assert.NotNil(t, sastScanManager.sastScannerResults) { diff --git a/xray/commands/audit/jas/secrets/secretsscanner.go b/xray/commands/audit/jas/secrets/secretsscanner.go index eb383f6f4..aad7fc6a6 100644 --- a/xray/commands/audit/jas/secrets/secretsscanner.go +++ b/xray/commands/audit/jas/secrets/secretsscanner.go @@ -12,8 +12,9 @@ import ( ) const ( - secretsScanCommand = "sec" - secretsScannerType = "secrets-scan" + secretsScanCommand = "sec" + secretsScannerType = "secrets-scan" + secretsDocsUrlSuffix = "secrets" ) type SecretScanManager struct { @@ -59,7 +60,7 @@ func (ssm *SecretScanManager) Run(module jfrogappsconfig.Module) (err error) { if err = ssm.runAnalyzerManager(); err != nil { return } - workingDirRuns, err := jas.ReadJasScanRunsFromFile(ssm.scanner.ResultsFileName, module.SourceRoot) + workingDirRuns, err := jas.ReadJasScanRunsFromFile(ssm.scanner.ResultsFileName, module.SourceRoot, secretsDocsUrlSuffix) if err != nil { return } diff --git a/xray/commands/audit/jas/secrets/secretsscanner_test.go b/xray/commands/audit/jas/secrets/secretsscanner_test.go index bd9ec826d..1b2a053ec 100644 --- a/xray/commands/audit/jas/secrets/secretsscanner_test.go +++ b/xray/commands/audit/jas/secrets/secretsscanner_test.go @@ -66,7 +66,7 @@ func TestParseResults_EmptyResults(t *testing.T) { // Act var err error - secretScanManager.secretsScannerResults, err = jas.ReadJasScanRunsFromFile(secretScanManager.scanner.ResultsFileName, scanner.JFrogAppsConfig.Modules[0].SourceRoot) + secretScanManager.secretsScannerResults, err = jas.ReadJasScanRunsFromFile(secretScanManager.scanner.ResultsFileName, scanner.JFrogAppsConfig.Modules[0].SourceRoot, secretsDocsUrlSuffix) // Assert if assert.NoError(t, err) && assert.NotNil(t, secretScanManager.secretsScannerResults) { @@ -89,7 +89,7 @@ func TestParseResults_ResultsContainSecrets(t *testing.T) { // Act var err error - secretScanManager.secretsScannerResults, err = jas.ReadJasScanRunsFromFile(secretScanManager.scanner.ResultsFileName, scanner.JFrogAppsConfig.Modules[0].SourceRoot) + secretScanManager.secretsScannerResults, err = jas.ReadJasScanRunsFromFile(secretScanManager.scanner.ResultsFileName, scanner.JFrogAppsConfig.Modules[0].SourceRoot, secretsDocsUrlSuffix) // Assert if assert.NoError(t, err) && assert.NotNil(t, secretScanManager.secretsScannerResults) { diff --git a/xray/commands/audit/jasrunner.go b/xray/commands/audit/jasrunner.go index 76a59ff8e..c528fdadc 100644 --- a/xray/commands/audit/jasrunner.go +++ b/xray/commands/audit/jasrunner.go @@ -52,9 +52,6 @@ func runJasScannersAndSetResults(scanResults *utils.ExtendedScanResults, directD if err != nil { return } - if !utils.IsSastSupported() { - return - } if progress != nil { progress.SetHeadlineMsg("Running SAST scanning") } diff --git a/xray/utils/analyzermanager.go b/xray/utils/analyzermanager.go index 9866692de..cf894d660 100644 --- a/xray/utils/analyzermanager.go +++ b/xray/utils/analyzermanager.go @@ -3,7 +3,6 @@ package utils import ( "errors" "fmt" - "github.com/jfrog/gofrog/version" "os" "os/exec" "path" @@ -24,7 +23,6 @@ const ( ApplicabilityFeatureId = "contextual_analysis" AnalyzerManagerZipName = "analyzerManager.zip" defaultAnalyzerManagerVersion = "1.3.2.2019257" - minAnalyzerManagerVersionForSast = "1.3" analyzerManagerDownloadPath = "xsc-gen-exe-analyzer-manager-local/v1" analyzerManagerDirName = "analyzerManager" analyzerManagerExecutableName = "analyzerManager" @@ -150,10 +148,6 @@ func GetAnalyzerManagerVersion() string { return defaultAnalyzerManagerVersion } -func IsSastSupported() bool { - return version.NewVersion(GetAnalyzerManagerVersion()).AtLeast(minAnalyzerManagerVersionForSast) -} - func GetAnalyzerManagerDirAbsolutePath() (string, error) { jfrogDir, err := config.GetJfrogDependenciesPath() if err != nil { diff --git a/xray/utils/resultwriter.go b/xray/utils/resultwriter.go index 9a97a7e87..553f37388 100644 --- a/xray/utils/resultwriter.go +++ b/xray/utils/resultwriter.go @@ -25,6 +25,8 @@ const ( Json OutputFormat = "json" SimpleJson OutputFormat = "simple-json" Sarif OutputFormat = "sarif" + + BaseDocumentationURL = "https://docs.jfrog-applications.jfrog.io/jfrog-security-features/" ) const MissingCveScore = "0" @@ -115,7 +117,11 @@ func (rw *ResultsWriter) PrintScanResults() error { case Json: return PrintJson(rw.results.getXrayScanResults()) case Sarif: - sarifFile, err := GenerateSarifContentFromResults(rw.results, rw.isMultipleRoots, rw.includeLicenses, false) + sarifReport, err := GenereateSarifReportFromResults(rw.results, rw.isMultipleRoots, rw.includeLicenses) + if err != nil { + return err + } + sarifFile, err := ConvertSarifReportToString(sarifReport) if err != nil { return err } @@ -153,9 +159,6 @@ func (rw *ResultsWriter) printScanResultsTables() (err error) { if err = PrintIacTable(rw.results.IacScanResults, rw.results.EntitledForJas); err != nil { return } - if !IsSastSupported() { - return - } return PrintSastTable(rw.results.SastScanResults, rw.results.EntitledForJas) } @@ -172,12 +175,12 @@ func printMessage(message string) { log.Output("💬" + message) } -func GenerateSarifContentFromResults(extendedResults *ExtendedScanResults, isMultipleRoots, includeLicenses, markdownOutput bool) (sarifStr string, err error) { - report, err := NewReport() +func GenereateSarifReportFromResults(extendedResults *ExtendedScanResults, isMultipleRoots, includeLicenses bool) (report *sarif.Report, err error) { + report, err = NewReport() if err != nil { return } - xrayRun, err := convertXrayResponsesToSarifRun(extendedResults, isMultipleRoots, includeLicenses, markdownOutput) + xrayRun, err := convertXrayResponsesToSarifRun(extendedResults, isMultipleRoots, includeLicenses) if err != nil { return } @@ -188,23 +191,26 @@ func GenerateSarifContentFromResults(extendedResults *ExtendedScanResults, isMul report.Runs = append(report.Runs, extendedResults.SecretsScanResults...) report.Runs = append(report.Runs, extendedResults.SastScanResults...) + return +} + +func ConvertSarifReportToString(report *sarif.Report) (sarifStr string, err error) { out, err := json.Marshal(report) if err != nil { return "", errorutils.CheckError(err) } - return clientUtils.IndentJson(out), nil } -func convertXrayResponsesToSarifRun(extendedResults *ExtendedScanResults, isMultipleRoots, includeLicenses, markdownOutput bool) (run *sarif.Run, err error) { +func convertXrayResponsesToSarifRun(extendedResults *ExtendedScanResults, isMultipleRoots, includeLicenses bool) (run *sarif.Run, err error) { xrayJson, err := convertXrayScanToSimpleJson(extendedResults, isMultipleRoots, includeLicenses, true) if err != nil { return } - xrayRun := sarif.NewRunWithInformationURI("JFrog Xray Sca", "https://jfrog.com/xray/") + xrayRun := sarif.NewRunWithInformationURI("JFrog Xray SCA", BaseDocumentationURL+"sca") xrayRun.Tool.Driver.Version = &extendedResults.XrayVersion if len(xrayJson.Vulnerabilities) > 0 || len(xrayJson.SecurityViolations) > 0 { - if err = extractXrayIssuesToSarifRun(xrayRun, xrayJson, markdownOutput); err != nil { + if err = extractXrayIssuesToSarifRun(xrayRun, xrayJson); err != nil { return } } @@ -212,20 +218,19 @@ func convertXrayResponsesToSarifRun(extendedResults *ExtendedScanResults, isMult return } -func extractXrayIssuesToSarifRun(run *sarif.Run, xrayJson formats.SimpleJsonResults, markdownOutput bool) error { +func extractXrayIssuesToSarifRun(run *sarif.Run, xrayJson formats.SimpleJsonResults) error { for _, vulnerability := range xrayJson.Vulnerabilities { if err := addXrayCveIssueToSarifRun( vulnerability.Cves, vulnerability.IssueId, vulnerability.Severity, - vulnerability.Technology.GetPackageDescriptor(), + vulnerability.Technology, vulnerability.Components, vulnerability.Applicable, vulnerability.ImpactedDependencyName, vulnerability.ImpactedDependencyVersion, vulnerability.Summary, vulnerability.FixedVersions, - markdownOutput, run, ); err != nil { return err @@ -236,14 +241,13 @@ func extractXrayIssuesToSarifRun(run *sarif.Run, xrayJson formats.SimpleJsonResu violation.Cves, violation.IssueId, violation.Severity, - violation.Technology.GetPackageDescriptor(), + violation.Technology, violation.Components, violation.Applicable, violation.ImpactedDependencyName, violation.ImpactedDependencyVersion, violation.Summary, violation.FixedVersions, - markdownOutput, run, ); err != nil { return err @@ -258,35 +262,66 @@ func extractXrayIssuesToSarifRun(run *sarif.Run, xrayJson formats.SimpleJsonResu return nil } -func addXrayCveIssueToSarifRun(cves []formats.CveRow, issueId, severity, file string, components []formats.ComponentRow, applicable, impactedDependencyName, impactedDependencyVersion, summary string, fixedVersions []string, markdownOutput bool, run *sarif.Run) error { +func addXrayCveIssueToSarifRun(cves []formats.CveRow, issueId, severity string, tech coreutils.Technology, components []formats.ComponentRow, applicable, impactedDependencyName, impactedDependencyVersion, summary string, fixedVersions []string, run *sarif.Run) error { maxCveScore, err := findMaxCVEScore(cves) if err != nil { return err } cveId := GetIssueIdentifier(cves, issueId) msg := getVulnerabilityOrViolationSarifHeadline(impactedDependencyName, impactedDependencyVersion, cveId) - location := sarif.NewLocation().WithPhysicalLocation(sarif.NewPhysicalLocation().WithArtifactLocation(sarif.NewArtifactLocation().WithUri(file))) - + location, err := getXrayIssueLocationIfValidExists(tech, run) + if err != nil { + return err + } if rule, isNewRule := addResultToSarifRun(cveId, msg, severity, location, run); isNewRule { cveRuleProperties := sarif.NewPropertyBag() if maxCveScore != MissingCveScore { cveRuleProperties.Add("security-severity", maxCveScore) } rule.WithProperties(cveRuleProperties.Properties) - if markdownOutput { - formattedDirectDependencies, err := getDirectDependenciesFormatted(components) - if err != nil { - return err - } - markdownDescription := getSarifTableDescription(formattedDirectDependencies, maxCveScore, applicable, fixedVersions) + "\n" - rule.WithMarkdownHelp(markdownDescription) - } else { - rule.WithDescription(summary) + formattedDirectDependencies, err := getDirectDependenciesFormatted(components) + if err != nil { + return err } + markdownDescription := getSarifTableDescription(formattedDirectDependencies, maxCveScore, applicable, fixedVersions) + "\n" + rule.WithHelp(&sarif.MultiformatMessageString{ + Text: &summary, + Markdown: &markdownDescription, + }) } return nil } +func getDescriptorFullPath(tech coreutils.Technology, run *sarif.Run) (string, error) { + descriptors := tech.GetPackageDescriptor() + if len(descriptors) == 1 { + // Generate the full path + return GetFullLocationFileName(strings.TrimSpace(descriptors[0]), run.Invocations), nil + } + for _, descriptor := range descriptors { + // If multiple options return first to match + absolutePath := GetFullLocationFileName(strings.TrimSpace(descriptor), run.Invocations) + if exists, err := fileutils.IsFileExists(absolutePath, false); err != nil { + return "", err + } else if exists { + return absolutePath, nil + } + } + return "", nil +} + +// Get the descriptor location with the Xray issues if exists. +func getXrayIssueLocationIfValidExists(tech coreutils.Technology, run *sarif.Run) (location *sarif.Location, err error) { + descriptorPath, err := getDescriptorFullPath(tech, run) + if err != nil { + return + } + if strings.TrimSpace(descriptorPath) == "" { + return + } + return sarif.NewLocation().WithPhysicalLocation(sarif.NewPhysicalLocation().WithArtifactLocation(sarif.NewArtifactLocation().WithUri("file://" + descriptorPath))), nil +} + func addResultToSarifRun(issueId, msg, severity string, location *sarif.Location, run *sarif.Run) (rule *sarif.ReportingDescriptor, isNewRule bool) { if rule, _ = run.GetRuleById(issueId); rule == nil { isNewRule = true diff --git a/xray/utils/resultwriter_test.go b/xray/utils/resultwriter_test.go index 57458ea34..a67dfe86c 100644 --- a/xray/utils/resultwriter_test.go +++ b/xray/utils/resultwriter_test.go @@ -1,9 +1,14 @@ package utils import ( + "os" + "path/filepath" "testing" + "github.com/jfrog/jfrog-cli-core/v2/utils/coreutils" + "github.com/jfrog/jfrog-cli-core/v2/utils/tests" "github.com/jfrog/jfrog-cli-core/v2/xray/formats" + "github.com/owenrumney/go-sarif/v2/sarif" "github.com/stretchr/testify/assert" ) @@ -134,3 +139,57 @@ func TestFindMaxCVEScore(t *testing.T) { }) } } + +func TestGetXrayIssueLocationIfValidExists(t *testing.T) { + testDir, cleanup := tests.CreateTempDirWithCallbackAndAssert(t) + defer cleanup() + invocation := sarif.NewInvocation().WithWorkingDirectory(sarif.NewSimpleArtifactLocation(testDir)) + file, err := os.Create(filepath.Join(testDir, "go.mod")) + assert.NoError(t, err) + assert.NotNil(t, file) + defer func() { assert.NoError(t, file.Close()) }() + file2, err := os.Create(filepath.Join(testDir, "build.gradle.kts")) + assert.NoError(t, err) + assert.NotNil(t, file2) + defer func() { assert.NoError(t, file2.Close()) }() + + testCases := []struct { + name string + tech coreutils.Technology + run *sarif.Run + expectedOutput *sarif.Location + }{ + { + name: "No descriptor information", + tech: coreutils.Pip, + run: CreateRunWithDummyResults().WithInvocations([]*sarif.Invocation{invocation}), + expectedOutput: nil, + }, + { + name: "One descriptor information", + tech: coreutils.Go, + run: CreateRunWithDummyResults().WithInvocations([]*sarif.Invocation{invocation}), + expectedOutput: sarif.NewLocation().WithPhysicalLocation(sarif.NewPhysicalLocation().WithArtifactLocation(sarif.NewArtifactLocation().WithUri("file://" + filepath.Join(testDir, "go.mod")))), + }, + { + name: "One descriptor information - no invocation", + tech: coreutils.Go, + run: CreateRunWithDummyResults(), + expectedOutput: sarif.NewLocation().WithPhysicalLocation(sarif.NewPhysicalLocation().WithArtifactLocation(sarif.NewArtifactLocation().WithUri("file://go.mod"))), + }, + { + name: "Multiple descriptor information", + tech: coreutils.Gradle, + run: CreateRunWithDummyResults().WithInvocations([]*sarif.Invocation{invocation}), + expectedOutput: sarif.NewLocation().WithPhysicalLocation(sarif.NewPhysicalLocation().WithArtifactLocation(sarif.NewArtifactLocation().WithUri("file://" + filepath.Join(testDir, "build.gradle.kts")))), + }, + } + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + output, err := getXrayIssueLocationIfValidExists(tc.tech, tc.run) + if assert.NoError(t, err) { + assert.Equal(t, tc.expectedOutput, output) + } + }) + } +} diff --git a/xray/utils/sarifutils.go b/xray/utils/sarifutils.go index d0a959bfe..0da577140 100644 --- a/xray/utils/sarifutils.go +++ b/xray/utils/sarifutils.go @@ -147,7 +147,6 @@ func GetRelativeLocationFileName(location *sarif.Location, invocations []*sarif. if len(invocations) > 0 { wd = GetInvocationWorkingDirectory(invocations[0]) } - GetLocationFileName(location) filePath := GetLocationFileName(location) if filePath != "" { return ExtractRelativePath(filePath, wd) @@ -155,8 +154,15 @@ func GetRelativeLocationFileName(location *sarif.Location, invocations []*sarif. return "" } +func GetFullLocationFileName(relative string, invocations []*sarif.Invocation) string { + if len(invocations) == 0 { + return relative + } + return filepath.Join(GetInvocationWorkingDirectory(invocations[0]), relative) +} + func SetLocationFileName(location *sarif.Location, fileName string) { - if location != nil && location.PhysicalLocation != nil && location.PhysicalLocation.Region != nil && location.PhysicalLocation.Region.Snippet != nil { + if location != nil && location.PhysicalLocation != nil && location.PhysicalLocation.ArtifactLocation != nil { location.PhysicalLocation.ArtifactLocation.URI = &fileName } } @@ -259,64 +265,3 @@ func GetInvocationWorkingDirectory(invocation *sarif.Invocation) string { } return "" } - -func CreateRunWithDummyResults(results ...*sarif.Result) *sarif.Run { - run := sarif.NewRunWithInformationURI("", "") - for _, result := range results { - if result.RuleID != nil { - run.AddRule(*result.RuleID) - } - run.AddResult(result) - } - return run -} - -func CreateResultWithLocations(msg, ruleId, level string, locations ...*sarif.Location) *sarif.Result { - return &sarif.Result{ - Message: *sarif.NewTextMessage(msg), - Locations: locations, - Level: &level, - RuleID: &ruleId, - } -} - -func CreateLocation(fileName string, startLine, startCol, endLine, endCol int, snippet string) *sarif.Location { - return &sarif.Location{ - PhysicalLocation: &sarif.PhysicalLocation{ - ArtifactLocation: &sarif.ArtifactLocation{URI: &fileName}, - Region: &sarif.Region{ - StartLine: &startLine, - StartColumn: &startCol, - EndLine: &endLine, - EndColumn: &endCol, - Snippet: &sarif.ArtifactContent{Text: &snippet}}}, - } -} - -func CreateDummyPassingResult(ruleId string) *sarif.Result { - kind := "pass" - return &sarif.Result{ - Kind: &kind, - RuleID: &ruleId, - } -} - -func CreateResultWithOneLocation(fileName string, startLine, startCol, endLine, endCol int, snippet, ruleId, level string) *sarif.Result { - return CreateResultWithLocations("", ruleId, level, CreateLocation(fileName, startLine, startCol, endLine, endCol, snippet)) -} - -func CreateCodeFlow(threadFlows ...*sarif.ThreadFlow) *sarif.CodeFlow { - flow := sarif.NewCodeFlow() - for _, threadFlow := range threadFlows { - flow.AddThreadFlow(threadFlow) - } - return flow -} - -func CreateThreadFlow(locations ...*sarif.Location) *sarif.ThreadFlow { - stackStrace := sarif.NewThreadFlow() - for _, location := range locations { - stackStrace.AddLocation(sarif.NewThreadFlowLocation().WithLocation(location)) - } - return stackStrace -} diff --git a/xray/utils/sarifutils_test.go b/xray/utils/sarifutils_test.go index 20a92de41..33358fe7a 100644 --- a/xray/utils/sarifutils_test.go +++ b/xray/utils/sarifutils_test.go @@ -1,6 +1,7 @@ package utils import ( + "path/filepath" "testing" "github.com/owenrumney/go-sarif/v2/sarif" @@ -294,6 +295,32 @@ func TestGetRelativeLocationFileName(t *testing.T) { } } +func TestGetFullLocationFileName(t *testing.T) { + tests := []struct { + file string + invocations []*sarif.Invocation + expectedOutput string + }{ + { + file: filepath.Join("root", "someDir", "another", "file"), + invocations: []*sarif.Invocation{}, + expectedOutput: filepath.Join("root", "someDir", "another", "file"), + }, + { + file: filepath.Join("another", "file"), + invocations: []*sarif.Invocation{ + {WorkingDirectory: sarif.NewSimpleArtifactLocation(filepath.Join("root", "someDir"))}, + {WorkingDirectory: sarif.NewSimpleArtifactLocation(filepath.Join("not", "relevant"))}, + }, + expectedOutput: filepath.Join("root", "someDir", "another", "file"), + }, + } + + for _, test := range tests { + assert.Equal(t, test.expectedOutput, GetFullLocationFileName(test.file, test.invocations)) + } +} + func TestSetLocationFileName(t *testing.T) { tests := []struct { location *sarif.Location diff --git a/xray/utils/test_sarifutils.go b/xray/utils/test_sarifutils.go new file mode 100644 index 000000000..5034b1eb6 --- /dev/null +++ b/xray/utils/test_sarifutils.go @@ -0,0 +1,64 @@ +package utils + +import "github.com/owenrumney/go-sarif/v2/sarif" + +func CreateRunWithDummyResults(results ...*sarif.Result) *sarif.Run { + run := sarif.NewRunWithInformationURI("", "") + for _, result := range results { + if result.RuleID != nil { + run.AddRule(*result.RuleID) + } + run.AddResult(result) + } + return run +} + +func CreateResultWithLocations(msg, ruleId, level string, locations ...*sarif.Location) *sarif.Result { + return &sarif.Result{ + Message: *sarif.NewTextMessage(msg), + Locations: locations, + Level: &level, + RuleID: &ruleId, + } +} + +func CreateLocation(fileName string, startLine, startCol, endLine, endCol int, snippet string) *sarif.Location { + return &sarif.Location{ + PhysicalLocation: &sarif.PhysicalLocation{ + ArtifactLocation: &sarif.ArtifactLocation{URI: &fileName}, + Region: &sarif.Region{ + StartLine: &startLine, + StartColumn: &startCol, + EndLine: &endLine, + EndColumn: &endCol, + Snippet: &sarif.ArtifactContent{Text: &snippet}}}, + } +} + +func CreateDummyPassingResult(ruleId string) *sarif.Result { + kind := "pass" + return &sarif.Result{ + Kind: &kind, + RuleID: &ruleId, + } +} + +func CreateResultWithOneLocation(fileName string, startLine, startCol, endLine, endCol int, snippet, ruleId, level string) *sarif.Result { + return CreateResultWithLocations("", ruleId, level, CreateLocation(fileName, startLine, startCol, endLine, endCol, snippet)) +} + +func CreateCodeFlow(threadFlows ...*sarif.ThreadFlow) *sarif.CodeFlow { + flow := sarif.NewCodeFlow() + for _, threadFlow := range threadFlows { + flow.AddThreadFlow(threadFlow) + } + return flow +} + +func CreateThreadFlow(locations ...*sarif.Location) *sarif.ThreadFlow { + stackStrace := sarif.NewThreadFlow() + for _, location := range locations { + stackStrace.AddLocation(sarif.NewThreadFlowLocation().WithLocation(location)) + } + return stackStrace +} From 9469fc8de7669eb1a9a74ced71b5d958392d92e8 Mon Sep 17 00:00:00 2001 From: Robi Nino Date: Tue, 3 Oct 2023 11:39:15 +0300 Subject: [PATCH 13/16] Support Platform Access Token (#938) --- general/envsetup/envsetup.go | 10 +- general/token/accesstokencreate.go | 150 +++++++++++++++++++++++++++++ go.mod | 4 +- go.sum | 4 +- 4 files changed, 159 insertions(+), 9 deletions(-) create mode 100644 general/token/accesstokencreate.go diff --git a/general/envsetup/envsetup.go b/general/envsetup/envsetup.go index cbe844133..a7cc75943 100644 --- a/general/envsetup/envsetup.go +++ b/general/envsetup/envsetup.go @@ -27,10 +27,9 @@ import ( type OutputFormat string const ( - myJfrogEndPoint = "https://myjfrog-api.jfrog.com/api/v1/activation/cloud/cli/getStatus/" - syncSleepInterval = 5 * time.Second // 5 seconds - maxWaitMinutes = 30 * time.Minute // 30 minutes - nonExpiredTokenValue = 0 // Access Tokens with 0 expiration value are actually generated by Access with 1 year expiration. + myJfrogEndPoint = "https://myjfrog-api.jfrog.com/api/v1/activation/cloud/cli/getStatus/" + syncSleepInterval = 5 * time.Second // 5 seconds + maxWaitMinutes = 30 * time.Minute // 30 minutes // OutputFormat values Human OutputFormat = "human" @@ -265,7 +264,8 @@ func GenerateNewLongTermRefreshableAccessToken(server *config.ServerDetails) (er func createLongExpirationRefreshableTokenParams() *services.CreateTokenParams { params := services.CreateTokenParams{} - params.ExpiresIn = nonExpiredTokenValue + // Using the platform's default expiration (1 year by default). + params.ExpiresIn = nil params.Refreshable = clientUtils.Pointer(true) params.Audience = "*@*" return ¶ms diff --git a/general/token/accesstokencreate.go b/general/token/accesstokencreate.go new file mode 100644 index 000000000..e1a7c4226 --- /dev/null +++ b/general/token/accesstokencreate.go @@ -0,0 +1,150 @@ +package token + +import ( + "encoding/json" + rtUtils "github.com/jfrog/jfrog-cli-core/v2/artifactory/utils" + "github.com/jfrog/jfrog-cli-core/v2/utils/config" + "github.com/jfrog/jfrog-client-go/access/services" + "github.com/jfrog/jfrog-client-go/auth" + "github.com/jfrog/jfrog-client-go/utils/errorutils" + "strings" +) + +const ( + AdminScope = "applied-permissions/admin" + GroupsScopePrefix = "applied-permissions/groups:" +) + +type AccessTokenCreateCommand struct { + serverDetails *config.ServerDetails + username string + projectKey string + + scope string + groups string + grantAdmin bool + + expiry *uint + refreshable bool + description string + + audience string + includeReferenceToken bool + + response *auth.CreateTokenResponseData +} + +func NewAccessTokenCreateCommand() *AccessTokenCreateCommand { + return &AccessTokenCreateCommand{response: new(auth.CreateTokenResponseData)} +} + +func (atc *AccessTokenCreateCommand) SetServerDetails(serverDetails *config.ServerDetails) *AccessTokenCreateCommand { + atc.serverDetails = serverDetails + return atc +} + +func (atc *AccessTokenCreateCommand) SetUsername(username string) *AccessTokenCreateCommand { + atc.username = username + return atc +} + +func (atc *AccessTokenCreateCommand) SetProjectKey(projectKey string) *AccessTokenCreateCommand { + atc.projectKey = projectKey + return atc +} + +func (atc *AccessTokenCreateCommand) SetGroups(groups string) *AccessTokenCreateCommand { + atc.groups = groups + return atc +} + +func (atc *AccessTokenCreateCommand) SetScope(scope string) *AccessTokenCreateCommand { + atc.scope = scope + return atc +} + +func (atc *AccessTokenCreateCommand) SetGrantAdmin(grantAdmin bool) *AccessTokenCreateCommand { + atc.grantAdmin = grantAdmin + return atc +} + +func (atc *AccessTokenCreateCommand) SetExpiry(expiry *uint) *AccessTokenCreateCommand { + atc.expiry = expiry + return atc +} + +func (atc *AccessTokenCreateCommand) SetRefreshable(refreshable bool) *AccessTokenCreateCommand { + atc.refreshable = refreshable + return atc +} + +func (atc *AccessTokenCreateCommand) SetDescription(description string) *AccessTokenCreateCommand { + atc.description = description + return atc +} + +func (atc *AccessTokenCreateCommand) SetAudience(audience string) *AccessTokenCreateCommand { + atc.audience = audience + return atc +} + +func (atc *AccessTokenCreateCommand) SetIncludeReferenceToken(includeReferenceToken bool) *AccessTokenCreateCommand { + atc.includeReferenceToken = includeReferenceToken + return atc +} + +func (atc *AccessTokenCreateCommand) Response() ([]byte, error) { + content, err := json.Marshal(*atc.response) + return content, errorutils.CheckError(err) +} + +func (atc *AccessTokenCreateCommand) ServerDetails() (*config.ServerDetails, error) { + return atc.serverDetails, nil +} + +func (atc *AccessTokenCreateCommand) CommandName() string { + return "jf_access_token_create" +} + +func (atc *AccessTokenCreateCommand) Run() error { + servicesManager, err := rtUtils.CreateAccessServiceManager(atc.serverDetails, false) + if err != nil { + return err + } + + *atc.response, err = servicesManager.CreateAccessToken(atc.getTokenParams()) + return err +} + +func (atc *AccessTokenCreateCommand) getTokenParams() services.CreateTokenParams { + tokenParams := services.CreateTokenParams{} + + tokenParams.Username = strings.ToLower(atc.username) + tokenParams.ProjectKey = atc.projectKey + tokenParams.Scope = atc.getScope() + tokenParams.ExpiresIn = atc.expiry + tokenParams.Refreshable = &atc.refreshable + tokenParams.Description = atc.description + tokenParams.Audience = atc.audience + tokenParams.IncludeReferenceToken = &atc.includeReferenceToken + return tokenParams +} + +// If an explicit scope was provided, apply it. +// Otherwise, if admin or groups scopes were requested, construct scope from them (space separated). +// If no scopes were requested, leave scope empty to provide the default user scope. +func (atc *AccessTokenCreateCommand) getScope() string { + if atc.scope != "" { + return atc.scope + } + + var scopes []string + if atc.groups != "" { + scopes = append(scopes, GroupsScopePrefix+atc.groups) + } + + if atc.grantAdmin { + scopes = append(scopes, AdminScope) + } + return strings.Join(scopes, " ") +} diff --git a/go.mod b/go.mod index da2539daf..7046713fa 100644 --- a/go.mod +++ b/go.mod @@ -12,7 +12,7 @@ require ( github.com/google/uuid v1.3.1 github.com/gookit/color v1.5.4 github.com/jedib0t/go-pretty/v6 v6.4.7 - github.com/jfrog/build-info-go v1.9.10 + github.com/jfrog/build-info-go v1.9.11 github.com/jfrog/gofrog v1.3.0 github.com/jfrog/jfrog-apps-config v1.0.1 github.com/jfrog/jfrog-client-go v1.32.3 @@ -95,6 +95,6 @@ require ( gopkg.in/warnings.v0 v0.1.2 // indirect ) -replace github.com/jfrog/jfrog-client-go => github.com/jfrog/jfrog-client-go v1.28.1-0.20230928142526-622034e3f57b +replace github.com/jfrog/jfrog-client-go => github.com/jfrog/jfrog-client-go v1.28.1-0.20231003083451-568b46797866 replace github.com/jfrog/build-info-go => github.com/jfrog/build-info-go v1.8.9-0.20230928084830-478bd49f5d3e diff --git a/go.sum b/go.sum index 31b7169b1..d3e2ce58f 100644 --- a/go.sum +++ b/go.sum @@ -201,8 +201,8 @@ github.com/jfrog/gofrog v1.3.0 h1:o4zgsBZE4QyDbz2M7D4K6fXPTBJht+8lE87mS9bw7Gk= github.com/jfrog/gofrog v1.3.0/go.mod h1:IFMc+V/yf7rA5WZ74CSbXe+Lgf0iApEQLxRZVzKRUR0= github.com/jfrog/jfrog-apps-config v1.0.1 h1:mtv6k7g8A8BVhlHGlSveapqf4mJfonwvXYLipdsOFMY= github.com/jfrog/jfrog-apps-config v1.0.1/go.mod h1:8AIIr1oY9JuH5dylz2S6f8Ym2MaadPLR6noCBO4C22w= -github.com/jfrog/jfrog-client-go v1.28.1-0.20230928142526-622034e3f57b h1:WXxRXkWaxFRdMechZJ0pkzZWCI6HNqlxLzAfr3qXjP8= -github.com/jfrog/jfrog-client-go v1.28.1-0.20230928142526-622034e3f57b/go.mod h1:AePTNv5H1YSGycxiL+1jXHCzqu3rCGruVP7S0N+BEEo= +github.com/jfrog/jfrog-client-go v1.28.1-0.20231003083451-568b46797866 h1:0SWHyECx5QfCjQXf8hDzbyM94B78Dvzei7TvD9CpsCY= +github.com/jfrog/jfrog-client-go v1.28.1-0.20231003083451-568b46797866/go.mod h1:wtk8jhtdrlzYvo3LLIwOn0OrqoSm8J5TiMfZzHIwLe8= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= github.com/kevinburke/ssh_config v1.2.0 h1:x584FjTGwHzMwvHx18PXxbBVzfnxogHaAReU4gf13a4= From 241f3086595895f4d4280c9f6f608ee832484784 Mon Sep 17 00:00:00 2001 From: Asaf Ambar Date: Tue, 3 Oct 2023 12:56:41 +0300 Subject: [PATCH 14/16] Support curation npm tree calc by package-lock only (#951) --- go.mod | 2 +- go.sum | 4 ++-- xray/commands/audit/sca/java/javautils.go | 2 +- xray/commands/audit/sca/npm/npm.go | 23 +++++++++++++++++--- xray/commands/audit/sca/npm/npm_test.go | 2 +- xray/commands/audit/scarunner.go | 4 ++-- xray/commands/curation/curationaudit.go | 15 ++++++++++--- xray/utils/auditbasicparams.go | 26 +++++++++++++++++++++++ xray/utils/auditnpmparams.go | 25 ++++++++++++++++++++++ 9 files changed, 90 insertions(+), 13 deletions(-) create mode 100644 xray/utils/auditnpmparams.go diff --git a/go.mod b/go.mod index 7046713fa..9b4443abd 100644 --- a/go.mod +++ b/go.mod @@ -97,4 +97,4 @@ require ( replace github.com/jfrog/jfrog-client-go => github.com/jfrog/jfrog-client-go v1.28.1-0.20231003083451-568b46797866 -replace github.com/jfrog/build-info-go => github.com/jfrog/build-info-go v1.8.9-0.20230928084830-478bd49f5d3e +replace github.com/jfrog/build-info-go => github.com/jfrog/build-info-go v1.8.9-0.20231003094520-3a09931ceaa8 diff --git a/go.sum b/go.sum index d3e2ce58f..6019152ef 100644 --- a/go.sum +++ b/go.sum @@ -195,8 +195,8 @@ github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 h1:BQSFePA1RWJOl github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99/go.mod h1:1lJo3i6rXxKeerYnT8Nvf0QmHCRC1n8sfWVwXF2Frvo= github.com/jedib0t/go-pretty/v6 v6.4.7 h1:lwiTJr1DEkAgzljsUsORmWsVn5MQjt1BPJdPCtJ6KXE= github.com/jedib0t/go-pretty/v6 v6.4.7/go.mod h1:Ndk3ase2CkQbXLLNf5QDHoYb6J9WtVfmHZu9n8rk2xs= -github.com/jfrog/build-info-go v1.8.9-0.20230928084830-478bd49f5d3e h1:tWNlQScbapCz5/EBc+lKBBQcZ/3QLgM3tM3HBEtxCTs= -github.com/jfrog/build-info-go v1.8.9-0.20230928084830-478bd49f5d3e/go.mod h1:ujJ8XQZMdT2tMkLSMJNyDd1pCY+duwHdjV+9or9FLIg= +github.com/jfrog/build-info-go v1.8.9-0.20231003094520-3a09931ceaa8 h1:XaXReF1CKOr5oOXq5KkZDuHt3q9Y6pJeNCjezxZo2CM= +github.com/jfrog/build-info-go v1.8.9-0.20231003094520-3a09931ceaa8/go.mod h1:ujJ8XQZMdT2tMkLSMJNyDd1pCY+duwHdjV+9or9FLIg= github.com/jfrog/gofrog v1.3.0 h1:o4zgsBZE4QyDbz2M7D4K6fXPTBJht+8lE87mS9bw7Gk= github.com/jfrog/gofrog v1.3.0/go.mod h1:IFMc+V/yf7rA5WZ74CSbXe+Lgf0iApEQLxRZVzKRUR0= github.com/jfrog/jfrog-apps-config v1.0.1 h1:mtv6k7g8A8BVhlHGlSveapqf4mJfonwvXYLipdsOFMY= diff --git a/xray/commands/audit/sca/java/javautils.go b/xray/commands/audit/sca/java/javautils.go index 37009cb5b..942066965 100644 --- a/xray/commands/audit/sca/java/javautils.go +++ b/xray/commands/audit/sca/java/javautils.go @@ -136,7 +136,7 @@ func hasLoop(idsAdded []string, idToAdd string) bool { return false } -func BuildDependencyTree(params *xrayutils.AuditBasicParams, tech coreutils.Technology) ([]*xrayUtils.GraphNode, []string, error) { +func BuildDependencyTree(params xrayutils.AuditParams, tech coreutils.Technology) ([]*xrayUtils.GraphNode, []string, error) { serverDetails, err := params.ServerDetails() if err != nil { return nil, nil, err diff --git a/xray/commands/audit/sca/npm/npm.go b/xray/commands/audit/sca/npm/npm.go index 269c6993e..9c863f877 100644 --- a/xray/commands/audit/sca/npm/npm.go +++ b/xray/commands/audit/sca/npm/npm.go @@ -15,7 +15,7 @@ const ( ignoreScriptsFlag = "--ignore-scripts" ) -func BuildDependencyTree(npmArgs []string) (dependencyTrees []*xrayUtils.GraphNode, uniqueDeps []string, err error) { +func BuildDependencyTree(params utils.AuditParams) (dependencyTrees []*xrayUtils.GraphNode, uniqueDeps []string, err error) { currentDir, err := coreutils.GetWorkingDirectory() if err != nil { return @@ -28,10 +28,11 @@ func BuildDependencyTree(npmArgs []string) (dependencyTrees []*xrayUtils.GraphNo if err != nil { return } - npmArgs = addIgnoreScriptsFlag(npmArgs) + + treeDepsParam := createTreeDepsParam(params) // Calculate npm dependencies - dependenciesMap, err := biutils.CalculateDependenciesMap(npmExecutablePath, currentDir, packageInfo.BuildInfoModuleId(), npmArgs, log.Logger) + dependenciesMap, err := biutils.CalculateDependenciesMap(npmExecutablePath, currentDir, packageInfo.BuildInfoModuleId(), treeDepsParam, log.Logger) if err != nil { log.Info("Used npm version:", npmVersion.GetVersion()) return @@ -46,6 +47,22 @@ func BuildDependencyTree(npmArgs []string) (dependencyTrees []*xrayUtils.GraphNo return } +func createTreeDepsParam(params utils.AuditParams) biutils.NpmTreeDepListParam { + if params == nil { + return biutils.NpmTreeDepListParam{ + Args: addIgnoreScriptsFlag([]string{}), + } + } + npmTreeDepParam := biutils.NpmTreeDepListParam{ + Args: addIgnoreScriptsFlag(params.Args()), + } + if npmParams, ok := params.(utils.AuditNpmParams); ok { + npmTreeDepParam.IgnoreNodeModules = npmParams.NpmIgnoreNodeModules() + npmTreeDepParam.OverwritePackageLock = npmParams.NpmOverwritePackageLock() + } + return npmTreeDepParam +} + // Add the --ignore-scripts to prevent execution of npm scripts during npm install. func addIgnoreScriptsFlag(npmArgs []string) []string { if !slices.Contains(npmArgs, ignoreScriptsFlag) { diff --git a/xray/commands/audit/sca/npm/npm_test.go b/xray/commands/audit/sca/npm/npm_test.go index ff4d0aa01..36871d29a 100644 --- a/xray/commands/audit/sca/npm/npm_test.go +++ b/xray/commands/audit/sca/npm/npm_test.go @@ -116,6 +116,6 @@ func TestIgnoreScripts(t *testing.T) { // The package.json file contain a postinstall script running an "exit 1" command. // Without the "--ignore-scripts" flag, the test will fail. - _, _, err := BuildDependencyTree([]string{}) + _, _, err := BuildDependencyTree(nil) assert.NoError(t, err) } diff --git a/xray/commands/audit/scarunner.go b/xray/commands/audit/scarunner.go index 7a364076d..ae4a25b1a 100644 --- a/xray/commands/audit/scarunner.go +++ b/xray/commands/audit/scarunner.go @@ -133,7 +133,7 @@ func getDirectDependenciesFromTree(dependencyTrees []*xrayCmdUtils.GraphNode) [] return directDependencies.ToSlice() } -func GetTechDependencyTree(params *xrayutils.AuditBasicParams, tech coreutils.Technology) (flatTree *xrayCmdUtils.GraphNode, fullDependencyTrees []*xrayCmdUtils.GraphNode, err error) { +func GetTechDependencyTree(params xrayutils.AuditParams, tech coreutils.Technology) (flatTree *xrayCmdUtils.GraphNode, fullDependencyTrees []*xrayCmdUtils.GraphNode, err error) { logMessage := fmt.Sprintf("Calculating %s dependencies", tech.ToFormal()) log.Info(logMessage + "...") if params.Progress() != nil { @@ -149,7 +149,7 @@ func GetTechDependencyTree(params *xrayutils.AuditBasicParams, tech coreutils.Te case coreutils.Maven, coreutils.Gradle: fullDependencyTrees, uniqueDeps, err = java.BuildDependencyTree(params, tech) case coreutils.Npm: - fullDependencyTrees, uniqueDeps, err = npm.BuildDependencyTree(params.Args()) + fullDependencyTrees, uniqueDeps, err = npm.BuildDependencyTree(params) case coreutils.Yarn: fullDependencyTrees, uniqueDeps, err = yarn.BuildDependencyTree() case coreutils.Go: diff --git a/xray/commands/curation/curationaudit.go b/xray/commands/curation/curationaudit.go index 1ef10d1bb..e6bf649c0 100644 --- a/xray/commands/curation/curationaudit.go +++ b/xray/commands/curation/curationaudit.go @@ -111,13 +111,13 @@ type CurationAuditCommand struct { workingDirs []string OriginPath string parallelRequests int - *utils.AuditBasicParams + utils.AuditParams } func NewCurationAuditCommand() *CurationAuditCommand { return &CurationAuditCommand{ extractPoliciesRegex: regexp.MustCompile(extractPoliciesRegexTemplate), - AuditBasicParams: &utils.AuditBasicParams{}, + AuditParams: &utils.AuditBasicParams{}, } } @@ -192,8 +192,17 @@ func (ca *CurationAuditCommand) doCurateAudit(results map[string][]*PackageStatu return nil } +func (ca *CurationAuditCommand) getAuditParamsByTech(tech coreutils.Technology) utils.AuditParams { + if tech == coreutils.Npm { + return utils.AuditNpmParams{AuditParams: ca.AuditParams}. + SetNpmIgnoreNodeModules(true). + SetNpmOverwritePackageLock(true) + } + return ca.AuditParams +} + func (ca *CurationAuditCommand) auditTree(tech coreutils.Technology, results map[string][]*PackageStatus) error { - flattenGraph, fullDependenciesTree, err := audit.GetTechDependencyTree(ca.AuditBasicParams, tech) + flattenGraph, fullDependenciesTree, err := audit.GetTechDependencyTree(ca.getAuditParamsByTech(tech), tech) if err != nil { return err } diff --git a/xray/utils/auditbasicparams.go b/xray/utils/auditbasicparams.go index ff990e9fc..e5c739517 100644 --- a/xray/utils/auditbasicparams.go +++ b/xray/utils/auditbasicparams.go @@ -5,6 +5,32 @@ import ( ioUtils "github.com/jfrog/jfrog-client-go/utils/io" ) +type AuditParams interface { + DirectDependencies() []string + AppendDependenciesForApplicabilityScan(directDependencies []string) *AuditBasicParams + ServerDetails() (*config.ServerDetails, error) + SetServerDetails(serverDetails *config.ServerDetails) *AuditBasicParams + PipRequirementsFile() string + SetPipRequirementsFile(requirementsFile string) *AuditBasicParams + ExcludeTestDependencies() bool + SetExcludeTestDependencies(excludeTestDependencies bool) *AuditBasicParams + UseWrapper() bool + SetUseWrapper(useWrapper bool) *AuditBasicParams + InsecureTls() bool + SetInsecureTls(insecureTls bool) *AuditBasicParams + Technologies() []string + SetTechnologies(technologies []string) *AuditBasicParams + Progress() ioUtils.ProgressMgr + SetProgress(progress ioUtils.ProgressMgr) + Args() []string + SetNpmScope(depType string) *AuditBasicParams + OutputFormat() OutputFormat + DepsRepo() string + SetDepsRepo(depsRepo string) *AuditBasicParams + IgnoreConfigFile() bool + SetIgnoreConfigFile(ignoreConfigFile bool) *AuditBasicParams +} + type AuditBasicParams struct { serverDetails *config.ServerDetails outputFormat OutputFormat diff --git a/xray/utils/auditnpmparams.go b/xray/utils/auditnpmparams.go new file mode 100644 index 000000000..947c09c5a --- /dev/null +++ b/xray/utils/auditnpmparams.go @@ -0,0 +1,25 @@ +package utils + +type AuditNpmParams struct { + AuditParams + npmIgnoreNodeModules bool + npmOverwritePackageLock bool +} + +func (anp AuditNpmParams) SetNpmIgnoreNodeModules(ignoreNpmNodeModules bool) AuditNpmParams { + anp.npmIgnoreNodeModules = ignoreNpmNodeModules + return anp +} + +func (anp AuditNpmParams) SetNpmOverwritePackageLock(overwritePackageLock bool) AuditNpmParams { + anp.npmOverwritePackageLock = overwritePackageLock + return anp +} + +func (anp AuditNpmParams) NpmIgnoreNodeModules() bool { + return anp.npmIgnoreNodeModules +} + +func (anp AuditNpmParams) NpmOverwritePackageLock() bool { + return anp.npmOverwritePackageLock +} From 77cffb08ce51ac474863cd9abeb1466061636bfa Mon Sep 17 00:00:00 2001 From: Yahav Itzhak Date: Tue, 3 Oct 2023 15:11:46 +0300 Subject: [PATCH 15/16] Transfer - Add 1 minute timeout to all requests (#983) --- .../transferfiles/longpropertycheck.go | 13 ++++++----- artifactory/commands/transferfiles/utils.go | 2 +- artifactory/utils/storageinfo.go | 2 +- artifactory/utils/utils.go | 23 +++++++++++-------- go.mod | 2 +- go.sum | 4 ++-- 6 files changed, 26 insertions(+), 20 deletions(-) diff --git a/artifactory/commands/transferfiles/longpropertycheck.go b/artifactory/commands/transferfiles/longpropertycheck.go index 35b9f120f..6b067bf40 100644 --- a/artifactory/commands/transferfiles/longpropertycheck.go +++ b/artifactory/commands/transferfiles/longpropertycheck.go @@ -3,23 +3,24 @@ package transferfiles import ( "encoding/json" "fmt" + "io" + "sync" + "github.com/jfrog/gofrog/parallel" "github.com/jfrog/jfrog-cli-core/v2/artifactory/commands/transferfiles/api" cmdutils "github.com/jfrog/jfrog-cli-core/v2/artifactory/commands/utils" - "github.com/jfrog/jfrog-cli-core/v2/artifactory/utils" "github.com/jfrog/jfrog-cli-core/v2/utils/coreutils" "github.com/jfrog/jfrog-cli-core/v2/utils/progressbar" "github.com/jfrog/jfrog-client-go/artifactory" servicesUtils "github.com/jfrog/jfrog-client-go/artifactory/services/utils" clientutils "github.com/jfrog/jfrog-client-go/utils" "golang.org/x/exp/slices" - "io" - "sync" "github.com/jfrog/jfrog-client-go/utils/log" - "github.com/jfrog/jfrog-client-go/utils/errorutils" "time" + + "github.com/jfrog/jfrog-client-go/utils/errorutils" ) const ( @@ -112,7 +113,7 @@ func (lpc *LongPropertyCheck) ExecuteCheck(args cmdutils.RunArguments) (passed b // Returns the number of long properties found func (lpc *LongPropertyCheck) longPropertiesTaskProducer(progress *progressbar.TasksProgressBar, args cmdutils.RunArguments) int { // Init - serviceManager, err := utils.CreateServiceManagerWithContext(args.Context, args.ServerDetails, false, 0, retries, retriesWaitMilliSecs) + serviceManager, err := createTransferServiceManager(args.Context, args.ServerDetails) if err != nil { return 0 } @@ -174,7 +175,7 @@ func getSearchAllPropertiesQuery(pageNumber int) string { // We keep only the files that are at the requested repos and pass them at the files channel func createSearchPropertyTask(property Property, repos []string, args cmdutils.RunArguments, filesChan chan FileWithLongProperty, progress *progressbar.TasksProgressBar) parallel.TaskFunc { return func(threadId int) (err error) { - serviceManager, err := utils.CreateServiceManagerWithContext(args.Context, args.ServerDetails, false, 0, retries, retriesWaitMilliSecs) + serviceManager, err := createTransferServiceManager(args.Context, args.ServerDetails) if err != nil { return } diff --git a/artifactory/commands/transferfiles/utils.go b/artifactory/commands/transferfiles/utils.go index 31fcd5045..902bd5421 100644 --- a/artifactory/commands/transferfiles/utils.go +++ b/artifactory/commands/transferfiles/utils.go @@ -131,7 +131,7 @@ func (m *InterruptionErr) Error() string { } func createTransferServiceManager(ctx context.Context, serverDetails *config.ServerDetails) (artifactory.ArtifactoryServicesManager, error) { - return utils.CreateServiceManagerWithContext(ctx, serverDetails, false, 0, retries, retriesWaitMilliSecs) + return utils.CreateServiceManagerWithContext(ctx, serverDetails, false, 0, retries, retriesWaitMilliSecs, time.Minute) } func createSrcRtUserPluginServiceManager(ctx context.Context, sourceRtDetails *config.ServerDetails) (*srcUserPluginService, error) { diff --git a/artifactory/utils/storageinfo.go b/artifactory/utils/storageinfo.go index be46105cd..bb512bc6b 100644 --- a/artifactory/utils/storageinfo.go +++ b/artifactory/utils/storageinfo.go @@ -35,7 +35,7 @@ type StorageInfoManager struct { } func NewStorageInfoManager(ctx context.Context, serverDetails *config.ServerDetails) (*StorageInfoManager, error) { - serviceManager, err := CreateServiceManagerWithContext(ctx, serverDetails, false, 0, serviceManagerRetriesPerRequest, serviceManagerRetriesWaitPerRequestMilliSecs) + serviceManager, err := CreateServiceManagerWithContext(ctx, serverDetails, false, 0, serviceManagerRetriesPerRequest, serviceManagerRetriesWaitPerRequestMilliSecs, time.Minute) if err != nil { return nil, err } diff --git a/artifactory/utils/utils.go b/artifactory/utils/utils.go index f20ed3c01..3730505ef 100644 --- a/artifactory/utils/utils.go +++ b/artifactory/utils/utils.go @@ -4,6 +4,14 @@ import ( "context" "encoding/json" "errors" + "io" + "net/http" + "net/url" + "os" + "path" + "path/filepath" + "time" + "github.com/jfrog/build-info-go/build" "github.com/jfrog/jfrog-cli-core/v2/utils/config" "github.com/jfrog/jfrog-cli-core/v2/utils/coreutils" @@ -18,12 +26,6 @@ import ( "github.com/jfrog/jfrog-client-go/utils/errorutils" ioUtils "github.com/jfrog/jfrog-client-go/utils/io" "github.com/jfrog/jfrog-client-go/utils/log" - "io" - "net/http" - "net/url" - "os" - "path" - "path/filepath" ) func GetProjectDir(global bool) (string, error) { @@ -84,16 +86,16 @@ func GetEncryptedPasswordFromArtifactory(artifactoryAuth auth.ServiceDetails, in } func CreateServiceManager(serverDetails *config.ServerDetails, httpRetries, httpRetryWaitMilliSecs int, isDryRun bool) (artifactory.ArtifactoryServicesManager, error) { - return CreateServiceManagerWithContext(context.Background(), serverDetails, isDryRun, 0, httpRetries, httpRetryWaitMilliSecs) + return CreateServiceManagerWithContext(context.Background(), serverDetails, isDryRun, 0, httpRetries, httpRetryWaitMilliSecs, 0) } // Create a service manager with threads. // If the value sent for httpRetries is negative, the default will be used. func CreateServiceManagerWithThreads(serverDetails *config.ServerDetails, isDryRun bool, threads, httpRetries, httpRetryWaitMilliSecs int) (artifactory.ArtifactoryServicesManager, error) { - return CreateServiceManagerWithContext(context.Background(), serverDetails, isDryRun, threads, httpRetries, httpRetryWaitMilliSecs) + return CreateServiceManagerWithContext(context.Background(), serverDetails, isDryRun, threads, httpRetries, httpRetryWaitMilliSecs, 0) } -func CreateServiceManagerWithContext(context context.Context, serverDetails *config.ServerDetails, isDryRun bool, threads, httpRetries, httpRetryWaitMilliSecs int) (artifactory.ArtifactoryServicesManager, error) { +func CreateServiceManagerWithContext(context context.Context, serverDetails *config.ServerDetails, isDryRun bool, threads, httpRetries, httpRetryWaitMilliSecs int, timeout time.Duration) (artifactory.ArtifactoryServicesManager, error) { certsPath, err := coreutils.GetJfrogCertsDir() if err != nil { return nil, err @@ -115,6 +117,9 @@ func CreateServiceManagerWithContext(context context.Context, serverDetails *con if threads > 0 { configBuilder.SetThreads(threads) } + if timeout > 0 { + configBuilder.SetOverallRequestTimeout(timeout) + } serviceConfig, err := configBuilder.Build() if err != nil { return nil, err diff --git a/go.mod b/go.mod index 9b4443abd..187381527 100644 --- a/go.mod +++ b/go.mod @@ -95,6 +95,6 @@ require ( gopkg.in/warnings.v0 v0.1.2 // indirect ) -replace github.com/jfrog/jfrog-client-go => github.com/jfrog/jfrog-client-go v1.28.1-0.20231003083451-568b46797866 +replace github.com/jfrog/jfrog-client-go => github.com/jfrog/jfrog-client-go v1.28.1-0.20231003120621-90e9d7ea05e9 replace github.com/jfrog/build-info-go => github.com/jfrog/build-info-go v1.8.9-0.20231003094520-3a09931ceaa8 diff --git a/go.sum b/go.sum index 6019152ef..0bbc22700 100644 --- a/go.sum +++ b/go.sum @@ -201,8 +201,8 @@ github.com/jfrog/gofrog v1.3.0 h1:o4zgsBZE4QyDbz2M7D4K6fXPTBJht+8lE87mS9bw7Gk= github.com/jfrog/gofrog v1.3.0/go.mod h1:IFMc+V/yf7rA5WZ74CSbXe+Lgf0iApEQLxRZVzKRUR0= github.com/jfrog/jfrog-apps-config v1.0.1 h1:mtv6k7g8A8BVhlHGlSveapqf4mJfonwvXYLipdsOFMY= github.com/jfrog/jfrog-apps-config v1.0.1/go.mod h1:8AIIr1oY9JuH5dylz2S6f8Ym2MaadPLR6noCBO4C22w= -github.com/jfrog/jfrog-client-go v1.28.1-0.20231003083451-568b46797866 h1:0SWHyECx5QfCjQXf8hDzbyM94B78Dvzei7TvD9CpsCY= -github.com/jfrog/jfrog-client-go v1.28.1-0.20231003083451-568b46797866/go.mod h1:wtk8jhtdrlzYvo3LLIwOn0OrqoSm8J5TiMfZzHIwLe8= +github.com/jfrog/jfrog-client-go v1.28.1-0.20231003120621-90e9d7ea05e9 h1:Lzk1+t5IQ1C6d03cqXvepPKGyVq5FDV0wSDlfc+bLqI= +github.com/jfrog/jfrog-client-go v1.28.1-0.20231003120621-90e9d7ea05e9/go.mod h1:wtk8jhtdrlzYvo3LLIwOn0OrqoSm8J5TiMfZzHIwLe8= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= github.com/kevinburke/ssh_config v1.2.0 h1:x584FjTGwHzMwvHx18PXxbBVzfnxogHaAReU4gf13a4= From 55eb9cdba4be01bf77d88cb8afa0b2d487fed568 Mon Sep 17 00:00:00 2001 From: Eran Turgeman <81029514+eranturgeman@users.noreply.github.com> Date: Tue, 3 Oct 2023 17:51:15 +0300 Subject: [PATCH 16/16] Updating dependencies (#985) --- go.mod | 8 ++++---- go.sum | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/go.mod b/go.mod index 187381527..20607002b 100644 --- a/go.mod +++ b/go.mod @@ -12,10 +12,10 @@ require ( github.com/google/uuid v1.3.1 github.com/gookit/color v1.5.4 github.com/jedib0t/go-pretty/v6 v6.4.7 - github.com/jfrog/build-info-go v1.9.11 + github.com/jfrog/build-info-go v1.9.12 github.com/jfrog/gofrog v1.3.0 github.com/jfrog/jfrog-apps-config v1.0.1 - github.com/jfrog/jfrog-client-go v1.32.3 + github.com/jfrog/jfrog-client-go v1.34.0 github.com/magiconair/properties v1.8.7 github.com/manifoldco/promptui v0.9.0 github.com/owenrumney/go-sarif/v2 v2.2.2 @@ -95,6 +95,6 @@ require ( gopkg.in/warnings.v0 v0.1.2 // indirect ) -replace github.com/jfrog/jfrog-client-go => github.com/jfrog/jfrog-client-go v1.28.1-0.20231003120621-90e9d7ea05e9 +// replace github.com/jfrog/jfrog-client-go => github.com/jfrog/jfrog-client-go v1.28.1-0.20231003120621-90e9d7ea05e9 -replace github.com/jfrog/build-info-go => github.com/jfrog/build-info-go v1.8.9-0.20231003094520-3a09931ceaa8 +// replace github.com/jfrog/build-info-go => github.com/jfrog/build-info-go v1.8.9-0.20231003094520-3a09931ceaa8 diff --git a/go.sum b/go.sum index 0bbc22700..fc5614fdc 100644 --- a/go.sum +++ b/go.sum @@ -195,14 +195,14 @@ github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 h1:BQSFePA1RWJOl github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99/go.mod h1:1lJo3i6rXxKeerYnT8Nvf0QmHCRC1n8sfWVwXF2Frvo= github.com/jedib0t/go-pretty/v6 v6.4.7 h1:lwiTJr1DEkAgzljsUsORmWsVn5MQjt1BPJdPCtJ6KXE= github.com/jedib0t/go-pretty/v6 v6.4.7/go.mod h1:Ndk3ase2CkQbXLLNf5QDHoYb6J9WtVfmHZu9n8rk2xs= -github.com/jfrog/build-info-go v1.8.9-0.20231003094520-3a09931ceaa8 h1:XaXReF1CKOr5oOXq5KkZDuHt3q9Y6pJeNCjezxZo2CM= -github.com/jfrog/build-info-go v1.8.9-0.20231003094520-3a09931ceaa8/go.mod h1:ujJ8XQZMdT2tMkLSMJNyDd1pCY+duwHdjV+9or9FLIg= +github.com/jfrog/build-info-go v1.9.12 h1:EixG8LrRy1GN8oaYbmOcbqKDzsch5e7nUhk4Pwb+BAQ= +github.com/jfrog/build-info-go v1.9.12/go.mod h1:ujJ8XQZMdT2tMkLSMJNyDd1pCY+duwHdjV+9or9FLIg= github.com/jfrog/gofrog v1.3.0 h1:o4zgsBZE4QyDbz2M7D4K6fXPTBJht+8lE87mS9bw7Gk= github.com/jfrog/gofrog v1.3.0/go.mod h1:IFMc+V/yf7rA5WZ74CSbXe+Lgf0iApEQLxRZVzKRUR0= github.com/jfrog/jfrog-apps-config v1.0.1 h1:mtv6k7g8A8BVhlHGlSveapqf4mJfonwvXYLipdsOFMY= github.com/jfrog/jfrog-apps-config v1.0.1/go.mod h1:8AIIr1oY9JuH5dylz2S6f8Ym2MaadPLR6noCBO4C22w= -github.com/jfrog/jfrog-client-go v1.28.1-0.20231003120621-90e9d7ea05e9 h1:Lzk1+t5IQ1C6d03cqXvepPKGyVq5FDV0wSDlfc+bLqI= -github.com/jfrog/jfrog-client-go v1.28.1-0.20231003120621-90e9d7ea05e9/go.mod h1:wtk8jhtdrlzYvo3LLIwOn0OrqoSm8J5TiMfZzHIwLe8= +github.com/jfrog/jfrog-client-go v1.34.0 h1:G+H73T3blf8DQkNZJFIfqlAnF4g+qOFL+IRlbPwD6GY= +github.com/jfrog/jfrog-client-go v1.34.0/go.mod h1:o1YDnyk8j7CWly23dbQZI9j7mSYcBxrPSfBtkul4N+0= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= github.com/kevinburke/ssh_config v1.2.0 h1:x584FjTGwHzMwvHx18PXxbBVzfnxogHaAReU4gf13a4=