From 49240aac820d34eb5ce29c859cf74421fffcb8d2 Mon Sep 17 00:00:00 2001 From: AlvoBen <144705560+AlvoBen@users.noreply.github.com> Date: Mon, 5 Aug 2024 15:25:43 +0300 Subject: [PATCH] CLI | Update the Cli Calls to SCA Risk Management Service to use Export Service (AST-42958) (#812) * add functionality to export wrapper * add functionality to export wrapper * fix fixlink of packages * fix dependencyPathArray of packages * fix linter * fix linter * fix linter * optimize runtime * refactor addPackageInformation in result.go * refactor addPackageInformation in result.go * fix linter * increase timeout * refactor * delete risk-management api from cli * fix linter * fix linter * fix integration test compilation * refactor unused var * add comment for better understanding * refactor * add log * try solve trivy vuls * try solve trivy vuls * try solve trivy vuls * try solve trivy vuls * try solve trivy vuls * revert trivy changes * Update ci.yml * resolve conversation * refactor --------- Co-authored-by: AlvoBen --- cmd/main.go | 3 +- internal/commands/.scripts/up.sh | 2 +- internal/commands/result.go | 209 +++++++++++++++++-------- internal/commands/scan.go | 9 +- internal/params/binds.go | 1 - internal/params/envs.go | 1 - internal/params/keys.go | 1 - internal/services/export.go | 77 +++++++-- internal/wrappers/export-http.go | 49 +++++- internal/wrappers/export.go | 40 ++++- internal/wrappers/mock/export-mock.go | 10 +- internal/wrappers/mock/results-mock.go | 34 ---- internal/wrappers/results-http.go | 115 +------------- internal/wrappers/results.go | 2 - test/integration/util_command.go | 3 +- 15 files changed, 316 insertions(+), 240 deletions(-) diff --git a/cmd/main.go b/cmd/main.go index 8bc97ccd5..8e6c70cfb 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -35,7 +35,6 @@ func main() { applications := viper.GetString(params.ApplicationsPathKey) results := viper.GetString(params.ResultsPathKey) scanSummary := viper.GetString(params.ScanSummaryPathKey) - scaPackage := viper.GetString(params.ScaPackagePathKey) risksOverview := viper.GetString(params.RisksOverviewPathKey) scsScanOverview := viper.GetString(params.ScsScanOverviewPathKey) uploads := viper.GetString(params.UploadsPathKey) @@ -63,7 +62,7 @@ func main() { applicationsWrapper := wrappers.NewApplicationsHTTPWrapper(applications) risksOverviewWrapper := wrappers.NewHTTPRisksOverviewWrapper(risksOverview) scsScanOverviewWrapper := wrappers.NewHTTPScanOverviewWrapper(scsScanOverview) - resultsWrapper := wrappers.NewHTTPResultsWrapper(results, scaPackage, scanSummary) + resultsWrapper := wrappers.NewHTTPResultsWrapper(results, scanSummary) authWrapper := wrappers.NewAuthHTTPWrapper() resultsPredicatesWrapper := wrappers.NewResultsPredicatesHTTPWrapper() codeBashingWrapper := wrappers.NewCodeBashingHTTPWrapper(codebashing) diff --git a/internal/commands/.scripts/up.sh b/internal/commands/.scripts/up.sh index b3eafa3b0..a5a987ce9 100755 --- a/internal/commands/.scripts/up.sh +++ b/internal/commands/.scripts/up.sh @@ -4,4 +4,4 @@ wget https://sca-downloads.s3.amazonaws.com/cli/latest/ScaResolver-linux64.tar.g tar -xzvf ScaResolver-linux64.tar.gz -C /tmp rm -rf ScaResolver-linux64.tar.gz # ignore mock and wrappers packages, as they checked by integration tests -go test $(go list ./... | grep -v "mock" | grep -v "wrappers" | grep -v "bitbucketserver" | grep -v "logger") -timeout 940.000s -coverprofile cover.out \ No newline at end of file +go test $(go list ./... | grep -v "mock" | grep -v "wrappers" | grep -v "bitbucketserver" | grep -v "logger") -timeout 20m -coverprofile cover.out \ No newline at end of file diff --git a/internal/commands/result.go b/internal/commands/result.go index adde31e11..222b29361 100644 --- a/internal/commands/result.go +++ b/internal/commands/result.go @@ -105,6 +105,7 @@ const ( fixLabel = "fix" redundantLabel = "redundant" delayValueForReport = 10 + fixLinkPrefix = "https://devhub.checkmarx.com/cve-details/" ) var summaryFormats = []string{ @@ -1056,7 +1057,7 @@ func CreateScanReport( return err } if !scanPending { - results, err = ReadResults(resultsWrapper, scan, params) + results, err = ReadResults(resultsWrapper, exportWrapper, scan, params) if err != nil { return err } @@ -1291,6 +1292,7 @@ func createDirectory(targetPath string) error { func ReadResults( resultsWrapper wrappers.ResultsWrapper, + exportWrapper wrappers.ExportWrapper, scan *wrappers.ScanResponseModel, params map[string]string, ) (results *wrappers.ScanResultsCollection, err error) { @@ -1314,7 +1316,7 @@ func ReadResults( // Compute SAST results redundancy resultsModel = ComputeRedundantSastResults(resultsModel) } - resultsModel, err = enrichScaResults(resultsWrapper, scan, params, resultsModel) + resultsModel, err = enrichScaResults(exportWrapper, scan, resultsModel) if err != nil { return nil, err } @@ -1326,33 +1328,17 @@ func ReadResults( } func enrichScaResults( - resultsWrapper wrappers.ResultsWrapper, + exportWrapper wrappers.ExportWrapper, scan *wrappers.ScanResponseModel, - params map[string]string, resultsModel *wrappers.ScanResultsCollection, ) (*wrappers.ScanResultsCollection, error) { if slices.Contains(scan.Engines, commonParams.ScaType) { - // Get additional information to enrich sca results - scaPackageModel, errorModel, err := resultsWrapper.GetAllResultsPackageByScanID(params) - if errorModel != nil { - logger.PrintfIfVerbose("%s: CODE: %d, %s", failedListingResults, errorModel.Code, errorModel.Message) - return resultsModel, errors.Errorf("%s: CODE: %d, %s", failedListingResults, errorModel.Code, errorModel.Message) - } - if err != nil { - logger.PrintfIfVerbose("%s", failedListingResults) - return resultsModel, errors.Wrapf(err, "%s", failedListingResults) - } - // Get additional information to add the type information to the sca results - scaTypeModel, errorModel, err := resultsWrapper.GetAllResultsTypeByScanID(params) - if errorModel != nil { - logger.PrintfIfVerbose("%s: CODE: %d, %s", failedListingResults, errorModel.Code, errorModel.Message) - return resultsModel, errors.Errorf("%s: CODE: %d, %s", failedListingResults, errorModel.Code, errorModel.Message) - } + scaExportDetails, err := services.GetExportPackage(exportWrapper, scan.ID) if err != nil { - logger.PrintfIfVerbose("%s", failedListingResults) - return resultsModel, errors.Wrapf(err, "%s", failedListingResults) + return nil, errors.Wrapf(err, "%s", failedListingResults) } - // Enrich sca results + scaPackageModel := parseScaExportPackage(scaExportDetails.Packages) + scaTypeModel := parseExportScaVulnerability(scaExportDetails.ScaTypes) if scaPackageModel != nil { resultsModel = addPackageInformation(resultsModel, scaPackageModel, scaTypeModel) } @@ -1363,6 +1349,61 @@ func enrichScaResults( return resultsModel, nil } +func parseExportScaVulnerability(types []wrappers.ScaType) *[]wrappers.ScaTypeCollection { + var scaTypes []wrappers.ScaTypeCollection + for _, t := range types { + scaTypes = append(scaTypes, wrappers.ScaTypeCollection(t)) + } + return &scaTypes +} + +func parseScaExportPackage(packages []wrappers.ScaPackage) *[]wrappers.ScaPackageCollection { + var scaPackages []wrappers.ScaPackageCollection + for _, pkg := range packages { + pkg := pkg + scaPackages = append(scaPackages, wrappers.ScaPackageCollection{ + ID: pkg.ID, + Locations: pkg.Locations, + DependencyPathArray: parsePackagePathToDependencyPath(&pkg), + Outdated: pkg.Outdated, + IsDirectDependency: pkg.IsDirectDependency, + }) + } + return &scaPackages +} + +func parsePackagePathToDependencyPath(pkg *wrappers.ScaPackage) [][]wrappers.DependencyPath { + var dependencyPathArray [][]wrappers.DependencyPath + for _, path := range pkg.PackagePathArray { + var dependencyPath []wrappers.DependencyPath + for _, dep := range path { + dependencyPath = append(dependencyPath, wrappers.DependencyPath{ + ID: dep.ID, + Name: dep.Name, + Version: dep.Version, + }) + } + dependencyPathArray = append(dependencyPathArray, dependencyPath) + } + + // We are doing this to maintain the same structure that was in risk-management api response + // in risk-management, if the length of the dependency path array is 1, it will be the main package + // in export service, if there are no dependencies, the package path array will be empty + if len(dependencyPathArray) == 0 { + appendMainPackageToDependencyPath(&dependencyPathArray, pkg) + } + return dependencyPathArray +} + +func appendMainPackageToDependencyPath(dependencyPathArray *[][]wrappers.DependencyPath, pkg *wrappers.ScaPackage) { + *dependencyPathArray = append(*dependencyPathArray, []wrappers.DependencyPath{{ + ID: pkg.ID, + Locations: pkg.Locations, + Name: pkg.Name, + IsDevelopment: pkg.IsDevelopmentDependency, + }}) +} + func removeContainerResults(model *wrappers.ScanResultsCollection) *wrappers.ScanResultsCollection { var newResults []*wrappers.ScanResult for _, result := range model.Results { @@ -2273,52 +2314,98 @@ func addPackageInformation( scaPackageModel *[]wrappers.ScaPackageCollection, scaTypeModel *[]wrappers.ScaTypeCollection, ) *wrappers.ScanResultsCollection { - var currentID string + locationsByID, typesByCVE := buildAuxiliaryScaMaps(resultsModel, scaPackageModel, scaTypeModel) + scaPackageMap := buildScaPackageMap(*scaPackageModel) for _, result := range resultsModel.Results { - if !(result.Type == commonParams.ScaType) { - continue - } else { - currentID = result.ScanResultData.PackageIdentifier - const precision = 1 - var roundedScore = util.RoundFloat(result.VulnerabilityDetails.CvssScore, precision) - result.VulnerabilityDetails.CvssScore = roundedScore - // Add the sca type - result.ScaType = buildScaType(typesByCVE, result) - // Temporary code for client - result.State = buildScaState(typesByCVE, result) - for _, packages := range *scaPackageModel { - currentPackage := packages - if packages.ID == currentID { - for _, dependencyPath := range currentPackage.DependencyPathArray { - head := &dependencyPath[0] - head.Locations = locationsByID[head.ID] - head.SupportsQuickFix = len(dependencyPath) == 1 - for _, location := range locationsByID[head.ID] { - head.SupportsQuickFix = head.SupportsQuickFix && util.IsPackageFileSupported(*location) - } - currentPackage.SupportsQuickFix = currentPackage.SupportsQuickFix || head.SupportsQuickFix - if result.ID != "" { - currentPackage.FixLink = "https://devhub.checkmarx.com/cve-details/" + result.ID - } else { - currentPackage.FixLink = "" - } - } - if currentPackage.IsDirectDependency { - currentPackage.TypeOfDependency = directDependencyType - } else { - currentPackage.TypeOfDependency = indirectDependencyType - } - result.ScanResultData.ScaPackageCollection = ¤tPackage - break - } - } + if result.Type == commonParams.ScaType { + processResult(result, locationsByID, typesByCVE, scaPackageMap) } } + return resultsModel } +func processResult( + result *wrappers.ScanResult, + locationsByID map[string][]*string, + typesByCVE map[string]wrappers.ScaTypeCollection, + scaPackageMap map[string]wrappers.ScaPackageCollection, // Updated parameter +) { + const precision = 1 + + currentID := result.ScanResultData.PackageIdentifier + result.VulnerabilityDetails.CvssScore = util.RoundFloat(result.VulnerabilityDetails.CvssScore, precision) + result.ScaType = buildScaType(typesByCVE, result) + result.State = buildScaState(typesByCVE, result) + + updatePackages(result, scaPackageMap, locationsByID, currentID) +} + +func updatePackages( + result *wrappers.ScanResult, + scaPackageMap map[string]wrappers.ScaPackageCollection, + locationsByID map[string][]*string, + currentID string, +) { + packages, found := scaPackageMap[currentID] + if !found { + return + } + + updateDependencyPaths(packages.DependencyPathArray, locationsByID) + if !packages.SupportsQuickFix { + packages.SupportsQuickFix = hasQuickFix(packages.DependencyPathArray) + } + + if packages.IsDirectDependency { + packages.TypeOfDependency = directDependencyType + } else { + packages.TypeOfDependency = indirectDependencyType + } + + packages.FixLink = buildFixLink(result) + result.ScanResultData.ScaPackageCollection = &packages +} + +func hasQuickFix(dependencyPaths [][]wrappers.DependencyPath) bool { + for i := range dependencyPaths { + head := &dependencyPaths[i][0] + if head.SupportsQuickFix { + return true + } + } + return false +} + +func buildScaPackageMap(scaPackageModel []wrappers.ScaPackageCollection) map[string]wrappers.ScaPackageCollection { + scaPackageMap := make(map[string]wrappers.ScaPackageCollection) + for i := range scaPackageModel { + scaPackageMap[scaPackageModel[i].ID] = scaPackageModel[i] + } + return scaPackageMap +} + +func updateDependencyPaths(dependencyPaths [][]wrappers.DependencyPath, locationsByID map[string][]*string) { + for i := range dependencyPaths { + head := &dependencyPaths[i][0] + head.Locations = locationsByID[head.ID] + head.SupportsQuickFix = len(dependencyPaths[i]) == 1 + + for _, location := range locationsByID[head.ID] { + head.SupportsQuickFix = head.SupportsQuickFix && util.IsPackageFileSupported(*location) + } + } +} + +func buildFixLink(result *wrappers.ScanResult) string { + if result.ID != "" { + return fmt.Sprint(fixLinkPrefix, result.ID) + } + return "" +} + func filterViolatedRules(policyModel wrappers.PolicyResponseModel) *wrappers.PolicyResponseModel { i := 0 for _, policy := range policyModel.Policies { diff --git a/internal/commands/scan.go b/internal/commands/scan.go index 533b8d936..56dca2260 100644 --- a/internal/commands/scan.go +++ b/internal/commands/scan.go @@ -1648,7 +1648,7 @@ func runCreateScanCommand( return err } - err = applyThreshold(cmd, resultsWrapper, scanResponseModel, thresholdMap) + err = applyThreshold(cmd, resultsWrapper, exportWrapper, scanResponseModel, thresholdMap) if err != nil { return err } @@ -1898,6 +1898,7 @@ func createReportsAfterScan( func applyThreshold( cmd *cobra.Command, resultsWrapper wrappers.ResultsWrapper, + exportWrapper wrappers.ExportWrapper, scanResponseModel *wrappers.ScanResponseModel, thresholdMap map[string]int, ) error { @@ -1911,7 +1912,7 @@ func applyThreshold( params[commonParams.SastRedundancyFlag] = "" } - summaryMap, err := getSummaryThresholdMap(resultsWrapper, scanResponseModel, params) + summaryMap, err := getSummaryThresholdMap(resultsWrapper, exportWrapper, scanResponseModel, params) if err != nil { return err } @@ -1994,11 +1995,11 @@ func parseThresholdLimit(limit string) (engineName string, intLimit int, err err return engineName, intLimit, err } -func getSummaryThresholdMap(resultsWrapper wrappers.ResultsWrapper, scan *wrappers.ScanResponseModel, params map[string]string) ( +func getSummaryThresholdMap(resultsWrapper wrappers.ResultsWrapper, exportWrapper wrappers.ExportWrapper, scan *wrappers.ScanResponseModel, params map[string]string) ( map[string]int, error, ) { - results, err := ReadResults(resultsWrapper, scan, params) + results, err := ReadResults(resultsWrapper, exportWrapper, scan, params) if err != nil { return nil, err } diff --git a/internal/params/binds.go b/internal/params/binds.go index ff002adf0..5f4b3131b 100644 --- a/internal/params/binds.go +++ b/internal/params/binds.go @@ -21,7 +21,6 @@ var EnvVarsBinds = []struct { {ScanSummaryPathKey, ScanSummaryPathEnv, "api/scan-summary"}, {RisksOverviewPathKey, RisksOverviewPathEnv, "api/apisec/static/api/scan/%s/risks-overview"}, {ScsScanOverviewPathKey, ScsScanOverviewPathEnv, "api/micro-engines/scans/%s/scan-overview"}, - {ScaPackagePathKey, ScaPackagePathEnv, "api/sca/risk-management/risk-reports/"}, {SastResultsPathKey, SastResultsPathEnv, "api/sast-results"}, {SastResultsPredicatesPathKey, SastResultsPredicatesPathEnv, "api/sast-results-predicates"}, {KicsResultsPathKey, KicsResultsPathEnv, "api/kics-results"}, diff --git a/internal/params/envs.go b/internal/params/envs.go index eb2b27353..a776100f2 100644 --- a/internal/params/envs.go +++ b/internal/params/envs.go @@ -21,7 +21,6 @@ const ( ApplicationsPathEnv = "CX_APPLICATIONS_PATH" ResultsPathEnv = "CX_RESULTS_PATH" ScanSummaryPathEnv = "CX_SCAN_SUMMARY_PATH" - ScaPackagePathEnv = "CX_SCA_PACKAGE_PATH" RisksOverviewPathEnv = "CX_RISKS_OVERVIEW_PATH" ScsScanOverviewPathEnv = "CX_SCS_SCAN_OVERVIEW_PATH" SastResultsPathEnv = "CX_SAST_RESULTS_PATH" diff --git a/internal/params/keys.go b/internal/params/keys.go index fc8762180..8dcb84e95 100644 --- a/internal/params/keys.go +++ b/internal/params/keys.go @@ -53,7 +53,6 @@ var ( LogsEngineLogPathKey = strings.ToLower(LogsEngineLogPathEnv) SastResultsPredicatesPathKey = strings.ToLower(SastResultsPredicatesPathEnv) KicsResultsPredicatesPathKey = strings.ToLower(KicsResultsPredicatesPathEnv) - ScaPackagePathKey = strings.ToLower(ScaPackagePathEnv) DescriptionsPathKey = strings.ToLower(DescriptionsPathEnv) TenantConfigurationPathKey = strings.ToLower(TenantConfigurationPathEnv) ResultsPdfReportPathKey = strings.ToLower(ResultsPdfReportPathEnv) diff --git a/internal/services/export.go b/internal/services/export.go index c05fbc483..3fb593f81 100644 --- a/internal/services/export.go +++ b/internal/services/export.go @@ -1,10 +1,12 @@ package services import ( + "fmt" "log" "strings" "time" + "github.com/checkmarx/ast-cli/internal/logger" "github.com/checkmarx/ast-cli/internal/wrappers" "github.com/pkg/errors" ) @@ -12,12 +14,52 @@ import ( const ( DefaultSbomOption = "CycloneDxJson" exportingStatus = "Exporting" - delayValueForReport = 10 + delayValueForReport = 3 pendingStatus = "Pending" - completedStatus = "completed" + completedStatus = "Completed" pollingTimeout = 5 // minutes ) +func GetExportPackage(exportWrapper wrappers.ExportWrapper, scanID string) (*wrappers.ScaPackageCollectionExport, error) { + var scaPackageCollection = &wrappers.ScaPackageCollectionExport{ + Packages: []wrappers.ScaPackage{}, + ScaTypes: []wrappers.ScaType{}, + } + payload := &wrappers.ExportRequestPayload{ + ScanID: scanID, + FileFormat: "ScanReportJson", + ExportParameters: wrappers.ExportParameters{ + HideDevAndTestDependencies: true, + ShowOnlyEffectiveLicenses: true, + ExcludePackages: false, + ExcludeLicenses: true, + ExcludeVulnerabilities: false, + ExcludePolicies: true, + }, + } + + exportID, err := exportWrapper.InitiateExportRequest(payload) + if err != nil { + return nil, err + } + logger.PrintIfVerbose(fmt.Sprintf("Initiated export request for scanID %s. ExportID: %s", scanID, exportID)) + + exportResponse, err := pollForCompletion(exportWrapper, exportID.ExportID) + + if err != nil { + return nil, err + } + + if exportResponse != nil && strings.EqualFold(exportResponse.ExportStatus, completedStatus) && exportResponse.FileURL != "" { + scaPackageCollection, err = exportWrapper.GetScaPackageCollectionExport(exportResponse.FileURL) + if err != nil { + return nil, err + } + } + + return scaPackageCollection, nil +} + func ExportSbomResults( exportWrapper wrappers.ExportWrapper, targetFile string, @@ -30,17 +72,22 @@ func ExportSbomResults( } log.Println("Generating SBOM report with " + payload.FileFormat + " file format") - sbomresp, err := exportWrapper.GenerateSbomReport(payload) + sbomresp, err := exportWrapper.InitiateExportRequest(payload) if err != nil { - return err + return errors.Wrapf(err, "Failed downloading SBOM report") } pollingResp, err := pollForCompletion(exportWrapper, sbomresp.ExportID) if err != nil { - return err + return errors.Wrapf(err, "Failed downloading SBOM report") + } + + if pollingResp == nil { + log.Println("No results were found, skipping SBOM report generation") + return nil } - if err := exportWrapper.DownloadSbomReport(pollingResp.ExportID, targetFile); err != nil { + if err := exportWrapper.DownloadExportReport(pollingResp.ExportID, targetFile); err != nil { return errors.Wrapf(err, "Failed downloading SBOM report") } return nil @@ -67,22 +114,32 @@ func pollForCompletion(exportWrapper wrappers.ExportWrapper, exportID string) (* timeout := time.After(pollingTimeout * time.Minute) pollingResp := &wrappers.ExportPollingResponse{ExportStatus: exportingStatus} + logger.PrintIfVerbose("Polling for export report generation completion") + for pollingResp.ExportStatus == exportingStatus || pollingResp.ExportStatus == pendingStatus { select { case <-timeout: - return nil, errors.Errorf("SBOM generating failed - Timed out after 5 minutes") + return nil, errors.Errorf("export generating failed - Timed out after 5 minutes") default: - resp, err := exportWrapper.GetSbomReportStatus(exportID) + resp, err := exportWrapper.GetExportReportStatus(exportID) if err != nil { - return nil, errors.Wrapf(err, "failed getting SBOM report status") + return nil, errors.Wrapf(err, "failed getting export report status") } pollingResp = resp + logger.PrintIfVerbose("Export status: " + pollingResp.ExportStatus) time.Sleep(delayValueForReport * time.Second) } } + if pollingResp.ErrorMessage != "" { + if strings.Contains(pollingResp.ErrorMessage, "No results were found") { + return nil, nil + } + return nil, fmt.Errorf("export error: %s", pollingResp.ErrorMessage) + } + if !strings.EqualFold(pollingResp.ExportStatus, completedStatus) { - return nil, errors.Errorf("SBOM generating failed - Current status: %s", pollingResp.ExportStatus) + return nil, errors.Errorf("export generating failed - Current status: %s", pollingResp.ExportStatus) } return pollingResp, nil diff --git a/internal/wrappers/export-http.go b/internal/wrappers/export-http.go index d513c382f..b8d2bfa9d 100644 --- a/internal/wrappers/export-http.go +++ b/internal/wrappers/export-http.go @@ -9,6 +9,7 @@ import ( "net/http" "os" + "github.com/checkmarx/ast-cli/internal/logger" commonParams "github.com/checkmarx/ast-cli/internal/params" "github.com/spf13/viper" @@ -16,8 +17,18 @@ import ( ) type ExportRequestPayload struct { - ScanID string `json:"ScanID"` - FileFormat string `json:"FileFormat"` + ScanID string `json:"ScanID"` + FileFormat string `json:"FileFormat"` + ExportParameters ExportParameters `json:"ExportParameters,omitempty"` +} + +type ExportParameters struct { + HideDevAndTestDependencies bool `json:"hideDevAndTestDependencies"` + ShowOnlyEffectiveLicenses bool `json:"showOnlyEffectiveLicenses"` + ExcludePackages bool `json:"excludePackages"` + ExcludeLicenses bool `json:"excludeLicenses"` + ExcludeVulnerabilities bool `json:"excludeVulnerabilities"` + ExcludePolicies bool `json:"excludePolicies"` } type ExportResponse struct { @@ -28,6 +39,7 @@ type ExportPollingResponse struct { ExportID string `json:"exportId"` ExportStatus string `json:"exportStatus"` FileURL string `json:"fileUrl"` + ErrorMessage string `json:"errorMessage"` } type ExportHTTPWrapper struct { path string @@ -39,13 +51,13 @@ func NewExportHTTPWrapper(path string) ExportWrapper { } } -func (r *ExportHTTPWrapper) GenerateSbomReport(payload *ExportRequestPayload) (*ExportResponse, error) { +func (e *ExportHTTPWrapper) InitiateExportRequest(payload *ExportRequestPayload) (*ExportResponse, error) { clientTimeout := viper.GetUint(commonParams.ClientTimeoutKey) params, err := json.Marshal(payload) if err != nil { return nil, errors.Wrapf(err, "Failed to parse request body") } - path := fmt.Sprintf("%s/%s", r.path, "requests") + path := fmt.Sprintf("%s/%s", e.path, "requests") resp, err := SendHTTPRequestWithJSONContentType(http.MethodPost, path, bytes.NewBuffer(params), true, clientTimeout) if err != nil { return nil, err @@ -70,9 +82,9 @@ func (r *ExportHTTPWrapper) GenerateSbomReport(payload *ExportRequestPayload) (* } } -func (r *ExportHTTPWrapper) GetSbomReportStatus(reportID string) (*ExportPollingResponse, error) { +func (e *ExportHTTPWrapper) GetExportReportStatus(reportID string) (*ExportPollingResponse, error) { clientTimeout := viper.GetUint(commonParams.ClientTimeoutKey) - path := fmt.Sprintf("%s/%s", r.path, "requests") + path := fmt.Sprintf("%s/%s", e.path, "requests") params := map[string]string{"returnUrl": "true", "exportId": reportID} resp, err := SendPrivateHTTPRequestWithQueryParams(http.MethodGet, path, params, nil, clientTimeout) if err != nil { @@ -98,9 +110,9 @@ func (r *ExportHTTPWrapper) GetSbomReportStatus(reportID string) (*ExportPolling } } -func (r *ExportHTTPWrapper) DownloadSbomReport(reportID, targetFile string) error { +func (e *ExportHTTPWrapper) DownloadExportReport(reportID, targetFile string) error { clientTimeout := viper.GetUint(commonParams.ClientTimeoutKey) - customURL := fmt.Sprintf("%s/%s/%s/%s", r.path, "requests", reportID, "download") + customURL := fmt.Sprintf("%s/%s/%s/%s", e.path, "requests", reportID, "download") resp, err := SendHTTPRequest(http.MethodGet, customURL, http.NoBody, true, clientTimeout) if err != nil { return err @@ -127,3 +139,24 @@ func (r *ExportHTTPWrapper) DownloadSbomReport(reportID, targetFile string) erro log.Printf("Downloaded file: %s - %d bytes\n", targetFile, size) return nil } + +func (e *ExportHTTPWrapper) GetScaPackageCollectionExport(fileURL string) (*ScaPackageCollectionExport, error) { + accessToken, err := GetAccessToken() + if err != nil { + return nil, err + } + resp, err := SendHTTPRequestByFullURL(http.MethodGet, fileURL, http.NoBody, true, viper.GetUint(commonParams.ClientTimeoutKey), accessToken, true) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + var scaPackageCollection ScaPackageCollectionExport + if err := json.NewDecoder(resp.Body).Decode(&scaPackageCollection); err != nil { + return nil, err + } + + logger.PrintIfVerbose("Retrieved SCA package collection export successfully") + + return &scaPackageCollection, nil +} diff --git a/internal/wrappers/export.go b/internal/wrappers/export.go index 03973554f..474ec7ebf 100644 --- a/internal/wrappers/export.go +++ b/internal/wrappers/export.go @@ -1,7 +1,41 @@ package wrappers type ExportWrapper interface { - GenerateSbomReport(payload *ExportRequestPayload) (*ExportResponse, error) - GetSbomReportStatus(reportID string) (*ExportPollingResponse, error) - DownloadSbomReport(reportID, targetFile string) error + InitiateExportRequest(payload *ExportRequestPayload) (*ExportResponse, error) + GetExportReportStatus(reportID string) (*ExportPollingResponse, error) + DownloadExportReport(reportID, targetFile string) error + GetScaPackageCollectionExport(fileURL string) (*ScaPackageCollectionExport, error) +} + +type ScaPackageCollectionExport struct { + Packages []ScaPackage `json:"Packages,omitempty"` + ScaTypes []ScaType `json:"Vulnerabilities,omitempty"` +} + +type ScaPackage struct { + ID string `json:"Id,omitempty"` + Name string `json:"Name,omitempty"` + Locations []*string `json:"Locations,omitempty"` + PackagePathArray [][]PackagePath `json:"PackagePaths,omitempty"` + Outdated bool `json:"Outdated,omitempty"` + IsDirectDependency bool `json:"IsDirectDependency"` + IsDevelopmentDependency bool `json:"IsDevelopmentDependency"` + SupportsQuickFix bool + FixLink string + TypeOfDependency string +} + +type PackagePath struct { + ID string `json:"Id,omitempty"` + Name string `json:"Name,omitempty"` + Version string `json:"Version,omitempty"` + IsResolved bool `json:"IsResolved,omitempty"` + Locations []*string + SupportsQuickFix bool +} + +type ScaType struct { + ID string `json:"Id,omitempty"` + Type string `json:"Type,omitempty"` + IsIgnored bool `json:"IsIgnored,omitempty"` } diff --git a/internal/wrappers/mock/export-mock.go b/internal/wrappers/mock/export-mock.go index f695047bc..ac08a5b11 100644 --- a/internal/wrappers/mock/export-mock.go +++ b/internal/wrappers/mock/export-mock.go @@ -10,7 +10,7 @@ import ( type ExportMockWrapper struct{} // GenerateSbomReport mock for tests -func (*ExportMockWrapper) GenerateSbomReport(payload *wrappers.ExportRequestPayload) (*wrappers.ExportResponse, error) { +func (*ExportMockWrapper) InitiateExportRequest(payload *wrappers.ExportRequestPayload) (*wrappers.ExportResponse, error) { if payload.ScanID == "err-scan-id" { return nil, errors.New("error") } @@ -20,7 +20,7 @@ func (*ExportMockWrapper) GenerateSbomReport(payload *wrappers.ExportRequestPayl } // GetSbomReportStatus mock for tests -func (*ExportMockWrapper) GetSbomReportStatus(_ string) (*wrappers.ExportPollingResponse, error) { +func (*ExportMockWrapper) GetExportReportStatus(_ string) (*wrappers.ExportPollingResponse, error) { return &wrappers.ExportPollingResponse{ ExportID: "id1234", ExportStatus: "Completed", @@ -29,7 +29,7 @@ func (*ExportMockWrapper) GetSbomReportStatus(_ string) (*wrappers.ExportPolling } // DownloadSbomReport mock for tests -func (*ExportMockWrapper) DownloadSbomReport(_, targetFile string) error { +func (*ExportMockWrapper) DownloadExportReport(_, targetFile string) error { file, err := os.Create(targetFile) defer func() { err = file.Close() @@ -42,3 +42,7 @@ func (*ExportMockWrapper) DownloadSbomReport(_, targetFile string) error { } return nil } + +func (e *ExportMockWrapper) GetScaPackageCollectionExport(fileURL string) (*wrappers.ScaPackageCollectionExport, error) { + return &wrappers.ScaPackageCollectionExport{}, nil +} diff --git a/internal/wrappers/mock/results-mock.go b/internal/wrappers/mock/results-mock.go index ebd43c8aa..ed207b33f 100644 --- a/internal/wrappers/mock/results-mock.go +++ b/internal/wrappers/mock/results-mock.go @@ -8,40 +8,6 @@ import ( type ResultsMockWrapper struct{} -func (r ResultsMockWrapper) GetAllResultsTypeByScanID(params map[string]string) (*[]wrappers.ScaTypeCollection, *wrappers.WebError, error) { - const mock = "mock" - var scaTypes = []wrappers.ScaTypeCollection{{ - ID: mock, - Type: mock, - }, { - ID: mock, - Type: mock, - }} - return &scaTypes, nil, nil -} - -func (r ResultsMockWrapper) GetAllResultsPackageByScanID(params map[string]string) (*[]wrappers.ScaPackageCollection, *wrappers.WebError, error) { - const mock = "mock" - var dependencyPath = wrappers.DependencyPath{ID: mock, Name: mock, Version: mock, IsResolved: true, IsDevelopment: false, Locations: nil} - var dependencyArray = [][]wrappers.DependencyPath{{dependencyPath}} - - dependencyArray[0][0] = dependencyPath - var scaPackages = []wrappers.ScaPackageCollection{{ - ID: mock, - FixLink: mock, - Locations: nil, - DependencyPathArray: dependencyArray, - Outdated: false, - }, { - ID: mock, - FixLink: mock, - Locations: nil, - DependencyPathArray: dependencyArray, - Outdated: false, - }} - return &scaPackages, nil, nil -} - var containersResults = &wrappers.ScanResult{ Type: "containers", Severity: "medium", diff --git a/internal/wrappers/results-http.go b/internal/wrappers/results-http.go index 7205947ce..1ec0b274e 100644 --- a/internal/wrappers/results-http.go +++ b/internal/wrappers/results-http.go @@ -3,10 +3,8 @@ package wrappers import ( "encoding/json" "fmt" - "io" "net/http" - "github.com/checkmarx/ast-cli/internal/logger" commonParams "github.com/checkmarx/ast-cli/internal/params" "github.com/spf13/viper" @@ -14,28 +12,24 @@ import ( ) const ( - failedToParseGetResults = "Failed to parse list results" - failedTogetScanSummaries = "Failed to get scan summaries" - failedToParseScanSummaries = "Failed to parse scan summaries" - respStatusCode = "response status code %d" - sort = "sort" - sortResultsDefault = "-severity" - offset = "offset" - astAPIPageLen = 1000 - astAPIPagingValue = "1000" + failedToParseGetResults = "Failed to parse list results" + respStatusCode = "response status code %d" + sort = "sort" + sortResultsDefault = "-severity" + offset = "offset" + astAPIPageLen = 1000 + astAPIPagingValue = "1000" ) type ResultsHTTPWrapper struct { resultsPath string scanSummaryPath string - scaPackagePath string } -func NewHTTPResultsWrapper(path, scaPackagePath, scanSummaryPath string) ResultsWrapper { +func NewHTTPResultsWrapper(path, scanSummaryPath string) ResultsWrapper { return &ResultsHTTPWrapper{ resultsPath: path, scanSummaryPath: scanSummaryPath, - scaPackagePath: scaPackagePath, } } @@ -125,99 +119,6 @@ func getResultsByOffset(resultPath string, params map[string]string, clientTimeo return nil, false, nil, errors.Errorf(respStatusCode, resp.StatusCode) } } -func (r *ResultsHTTPWrapper) GetAllResultsPackageByScanID(params map[string]string) ( - *[]ScaPackageCollection, - *WebError, - error, -) { - clientTimeout := viper.GetUint(commonParams.ClientTimeoutKey) - // AST has a limit of 10000 results, this makes it get all of them - params[limit] = limitValue - resp, err := SendPrivateHTTPRequestWithQueryParams( - http.MethodGet, - r.scaPackagePath+params[commonParams.ScanIDQueryParam]+"/packages", - params, - http.NoBody, - clientTimeout, - ) - if err != nil { - return nil, nil, err - } - - defer func() { - _ = resp.Body.Close() - }() - - decoder := json.NewDecoder(resp.Body) - - switch resp.StatusCode { - case http.StatusOK: - var model []ScaPackageCollection - err = decoder.Decode(&model) - if err != nil { - return nil, nil, errors.Wrapf(err, failedToParseGetResults) - } - return &model, nil, nil - case http.StatusNotFound: // scan was not triggered with SCA type or SCA scan didn't start yet - logger.PrintIfVerbose("SCA packages for enrichment not found") - return nil, nil, nil - default: - responseData, _ := io.ReadAll(resp.Body) - responseString := string(responseData) - logger.PrintIfVerbose("Failed to get SCA packages " + responseString) - return nil, nil, errors.Errorf(respStatusCode, resp.StatusCode) - } -} - -func (r *ResultsHTTPWrapper) GetAllResultsTypeByScanID(params map[string]string) ( - *[]ScaTypeCollection, - *WebError, - error, -) { - clientTimeout := viper.GetUint(commonParams.ClientTimeoutKey) - // AST has a limit of 10000 results, this makes it get all of them - params[limit] = limitValue - resp, err := SendPrivateHTTPRequestWithQueryParams( - http.MethodGet, - r.scaPackagePath+params[commonParams.ScanIDQueryParam]+"/vulnerabilities", - params, - http.NoBody, - clientTimeout, - ) - if err != nil { - return nil, nil, err - } - - defer func() { - if err == nil { - _ = resp.Body.Close() - } - }() - - decoder := json.NewDecoder(resp.Body) - - switch resp.StatusCode { - case http.StatusBadRequest, http.StatusInternalServerError: - errorModel := WebError{} - err = decoder.Decode(&errorModel) - if err != nil { - return nil, nil, errors.Wrapf(err, failedToParseGetResults) - } - return nil, &errorModel, nil - case http.StatusOK: - var model []ScaTypeCollection - err = decoder.Decode(&model) - if err != nil { - return nil, nil, errors.Wrapf(err, failedToParseGetResults) - } - return &model, nil, nil - case http.StatusNotFound: // scan was not triggered with SCA type or SCA scan didn't start yet - logger.PrintIfVerbose("SCA types for enrichment not found") - return nil, nil, nil - default: - return nil, nil, errors.Errorf(respStatusCode, resp.StatusCode) - } -} func (r *ResultsHTTPWrapper) GetResultsURL(projectID string) (string, error) { accessToken, err := GetAccessToken() diff --git a/internal/wrappers/results.go b/internal/wrappers/results.go index 5aae15c85..80e2b2b1d 100644 --- a/internal/wrappers/results.go +++ b/internal/wrappers/results.go @@ -2,8 +2,6 @@ package wrappers type ResultsWrapper interface { GetAllResultsByScanID(params map[string]string) (*ScanResultsCollection, *WebError, error) - GetAllResultsPackageByScanID(params map[string]string) (*[]ScaPackageCollection, *WebError, error) - GetAllResultsTypeByScanID(params map[string]string) (*[]ScaTypeCollection, *WebError, error) GetResultsURL(projectID string) (string, error) } diff --git a/test/integration/util_command.go b/test/integration/util_command.go index e2af38a5d..8c91f45ec 100644 --- a/test/integration/util_command.go +++ b/test/integration/util_command.go @@ -65,7 +65,6 @@ func createASTIntegrationTestCommand(t *testing.T) *cobra.Command { projects := viper.GetString(params.ProjectsPathKey) results := viper.GetString(params.ResultsPathKey) scanSummmaryPath := viper.GetString(params.ScanSummaryPathKey) - scaPackage := viper.GetString(params.ScaPackagePathKey) risksOverview := viper.GetString(params.RisksOverviewPathKey) scsScanOverviewPath := viper.GetString(params.ScsScanOverviewPathKey) uploads := viper.GetString(params.UploadsPathKey) @@ -93,7 +92,7 @@ func createASTIntegrationTestCommand(t *testing.T) *cobra.Command { groupsWrapper := wrappers.NewHTTPGroupsWrapper(groups) uploadsWrapper := wrappers.NewUploadsHTTPWrapper(uploads) projectsWrapper := wrappers.NewHTTPProjectsWrapper(projects) - resultsWrapper := wrappers.NewHTTPResultsWrapper(results, scaPackage, scanSummmaryPath) + resultsWrapper := wrappers.NewHTTPResultsWrapper(results, scanSummmaryPath) risksOverviewWrapper := wrappers.NewHTTPRisksOverviewWrapper(risksOverview) scsScanOverviewWrapper := wrappers.NewHTTPScanOverviewWrapper(scsScanOverviewPath) authWrapper := wrappers.NewAuthHTTPWrapper()