From 09ad12ebc3fcdf8928afb8f079a5a5fe4b7c3dc5 Mon Sep 17 00:00:00 2001 From: Eyal Delarea Date: Mon, 22 May 2023 17:41:55 +0300 Subject: [PATCH 1/6] Fix indirect dependancies in Golang & Improve logging (#317) --- commands/createfixpullrequests.go | 180 +++++++---- commands/createfixpullrequests_test.go | 130 +------- commands/scanandfixrepos.go | 18 +- commands/testdata/indirect-projects/go/go.mod | 22 ++ .../testdata/indirect-projects/go/main.go | 10 + .../testdata/indirect-projects/maven/pom.xml | 67 ++++ .../indirect-projects/pip/requirements.txt | 1 + .../testdata/indirect-projects/pip/setup.py | 9 + .../testdata/indirect-projects/pipenv/Pipfile | 12 + .../indirect-projects/pipenv/Pipfile.lock | 59 ++++ .../indirect-projects/poetry/pyproject.toml | 13 + commands/utils/consts.go | 7 + commands/utils/git.go | 4 +- commands/utils/git_test.go | 40 +-- .../packagehandlers/commonPackageHandler.go | 59 ++++ .../packagehandlers/genericpackagehandler.go | 58 ---- .../utils/packagehandlers/gopackagehandler.go | 14 + .../packagehandlers/mavenpackagehandler.go | 18 +- .../packagehandlers/npmpackagehandler.go | 23 ++ .../packagehandlers/packagehandlers_test.go | 295 +++++++++++++++++- .../packagehandlers/pythonpackagehandler.go | 44 ++- .../packagehandlers/yarnpackagehandler.go | 23 ++ commands/utils/params.go | 8 +- commands/utils/utils.go | 45 ++- commands/utils/utils_test.go | 10 +- 25 files changed, 840 insertions(+), 329 deletions(-) create mode 100644 commands/testdata/indirect-projects/go/go.mod create mode 100644 commands/testdata/indirect-projects/go/main.go create mode 100644 commands/testdata/indirect-projects/maven/pom.xml create mode 100644 commands/testdata/indirect-projects/pip/requirements.txt create mode 100644 commands/testdata/indirect-projects/pip/setup.py create mode 100644 commands/testdata/indirect-projects/pipenv/Pipfile create mode 100644 commands/testdata/indirect-projects/pipenv/Pipfile.lock create mode 100644 commands/testdata/indirect-projects/poetry/pyproject.toml create mode 100644 commands/utils/packagehandlers/commonPackageHandler.go delete mode 100644 commands/utils/packagehandlers/genericpackagehandler.go create mode 100644 commands/utils/packagehandlers/gopackagehandler.go create mode 100644 commands/utils/packagehandlers/npmpackagehandler.go create mode 100644 commands/utils/packagehandlers/yarnpackagehandler.go diff --git a/commands/createfixpullrequests.go b/commands/createfixpullrequests.go index 0b0d05fe6..fd358b00f 100644 --- a/commands/createfixpullrequests.go +++ b/commands/createfixpullrequests.go @@ -2,6 +2,7 @@ package commands import ( "context" + "errors" "fmt" "github.com/jfrog/frogbot/commands/utils" "github.com/jfrog/frogbot/commands/utils/packagehandlers" @@ -110,11 +111,11 @@ func (cfp *CreateFixPullRequestsCmd) fixImpactedPackagesAndCreatePRs(scanResults return nil } - log.Info("Found", len(fixVersionsMap), "vulnerable dependencies with fix versions") + log.Debug("Found", len(fixVersionsMap), "vulnerable dependencies with fix versions") return cfp.fixVulnerablePackages(fixVersionsMap) } -func (cfp *CreateFixPullRequestsCmd) fixVulnerablePackages(fixVersionsMap map[string]*utils.FixVersionInfo) (err error) { +func (cfp *CreateFixPullRequestsCmd) fixVulnerablePackages(fixVersionsMap map[string]*utils.FixDetails) (err error) { cfp.gitManager, err = utils.NewGitManager(cfp.dryRun, cfp.dryRunRepoPath, ".", "origin", cfp.details.Token, cfp.details.Username, cfp.details.Git) if err != nil { return @@ -134,7 +135,6 @@ func (cfp *CreateFixPullRequestsCmd) fixVulnerablePackages(fixVersionsMap map[st } } }() - if cfp.aggregateFixes { err = cfp.fixIssuesSinglePR(fixVersionsMap) } else { @@ -143,89 +143,126 @@ func (cfp *CreateFixPullRequestsCmd) fixVulnerablePackages(fixVersionsMap map[st return } -func (cfp *CreateFixPullRequestsCmd) fixIssuesSeparatePRs(fixVersionsMap map[string]*utils.FixVersionInfo) (err error) { - for impactedPackage, fixVersionInfo := range fixVersionsMap { - if err = cfp.fixSinglePackageAndCreatePR(impactedPackage, fixVersionInfo); err != nil { - log.Warn(err) +func (cfp *CreateFixPullRequestsCmd) fixIssuesSeparatePRs(fixVersionsMap map[string]*utils.FixDetails) (err error) { + var errList strings.Builder + if len(fixVersionsMap) == 0 { + return + } + log.Info("-----------------------------------------------------------------") + for _, fixVersionInfo := range fixVersionsMap { + if err = cfp.fixSinglePackageAndCreatePR(fixVersionInfo); err != nil { + cfp.handleUpdatePackageErrors(err, errList) } // After finishing to work on the current vulnerability, we go back to the base branch to start the next vulnerability fix - log.Info("Running git checkout to base branch:", cfp.details.Branch()) + log.Debug("Running git checkout to base branch:", cfp.details.Branch()) if err = cfp.gitManager.Checkout(cfp.details.Branch()); err != nil { return } } + logAppendedErrorsIfExists(errList) + log.Info("-----------------------------------------------------------------") return } -func (cfp *CreateFixPullRequestsCmd) fixIssuesSinglePR(fixVersionsMap map[string]*utils.FixVersionInfo) (err error) { +func (cfp *CreateFixPullRequestsCmd) fixIssuesSinglePR(fixVersionsMap map[string]*utils.FixDetails) (err error) { + var errList strings.Builder + var atLeastOneFix bool log.Info("-----------------------------------------------------------------") - log.Info("Start aggregated packages fix") + log.Info("Starting aggregated dependencies fix") aggregatedFixBranchName, err := cfp.gitManager.GenerateAggregatedFixBranchName(fixVersionsMap) if err != nil { return } - if err = cfp.createFixBranch(aggregatedFixBranchName, false); err != nil { + log.Debug("Creating branch", aggregatedFixBranchName, "...") + if err = cfp.gitManager.CreateBranchAndCheckout(aggregatedFixBranchName); err != nil { return } - // Fix all packages in the same branch - for impactedPackage, fixVersionInfo := range fixVersionsMap { - if err = cfp.updatePackageToFixedVersion(impactedPackage, fixVersionInfo); err != nil { - log.Error("could not fix impacted package", impactedPackage, "as part of the PR. Skipping it. Cause:", err.Error()) + // Fix all packages in the same branch if expected error accrued, log and continue. + for _, fixDetails := range fixVersionsMap { + if err := cfp.updatePackageToFixedVersion(fixDetails); err != nil { + cfp.handleUpdatePackageErrors(err, errList) + } else { + log.Info(fmt.Sprintf("Updated dependency '%s' to version '%s'", fixDetails.ImpactedDependency, fixDetails.FixVersion)) + atLeastOneFix = true } } - - if err = cfp.openAggregatedPullRequest(aggregatedFixBranchName); err != nil { - return fmt.Errorf("failed while creating aggreagted pull request. Error: \n%s", err.Error()) + if atLeastOneFix { + if err = cfp.openAggregatedPullRequest(aggregatedFixBranchName); err != nil { + return fmt.Errorf("failed while creating aggreagted pull request. Error: \n%s", err.Error()) + } } + logAppendedErrorsIfExists(errList) + log.Info("-----------------------------------------------------------------") return } -func (cfp *CreateFixPullRequestsCmd) fixSinglePackageAndCreatePR(impactedPackage string, fixVersionInfo *utils.FixVersionInfo) (err error) { - log.Info("-----------------------------------------------------------------") - log.Info("Start fixing", impactedPackage, "with", fixVersionInfo.FixVersion) - fixBranchName, err := cfp.gitManager.GenerateFixBranchName(cfp.details.Branch(), impactedPackage, fixVersionInfo.FixVersion) +// Handles possible error of update package operation +// When the expected custom error occurs, log to debug. +// else, append to errList string +func (cfp *CreateFixPullRequestsCmd) handleUpdatePackageErrors(err error, errList strings.Builder) { + if _, isCustomError := err.(*utils.ErrUnsupportedFix); isCustomError { + log.Debug(err.Error()) + } else { + errList.WriteString(err.Error() + "\n") + } +} + +// Creates a branch for the fixed package and open pull request against the target branch. +// In case branch already exists on remote, we skip it. +func (cfp *CreateFixPullRequestsCmd) fixSinglePackageAndCreatePR(fixDetails *utils.FixDetails) (err error) { + fixVersion := fixDetails.FixVersion + log.Debug("Start fixing", fixDetails.ImpactedDependency, "with", fixVersion) + fixBranchName, err := cfp.gitManager.GenerateFixBranchName(cfp.details.Branch(), fixDetails.ImpactedDependency, fixVersion) if err != nil { return } - err = cfp.createFixBranch(fixBranchName, true) + existsInRemote, err := cfp.gitManager.BranchExistsInRemote(fixBranchName) if err != nil { + return + } + if existsInRemote { + log.Info(fmt.Sprintf("A Pull Request updating dependency '%s' to version '%s' already exists.", fixDetails.ImpactedDependency, fixDetails.FixVersion)) + return + } + log.Debug("Creating branch", fixBranchName, "...") + if err = cfp.gitManager.CreateBranchAndCheckout(fixBranchName); err != nil { return fmt.Errorf("failed while creating new branch: \n%s", err.Error()) } - - if err = cfp.updatePackageToFixedVersion(impactedPackage, fixVersionInfo); err != nil { - return fmt.Errorf("failed while fixing %s to version: %s with error: \n%s", impactedPackage, fixVersionInfo.FixVersion, err.Error()) + if err = cfp.updatePackageToFixedVersion(fixDetails); err != nil { + return } - - if err = cfp.openFixingPullRequest(impactedPackage, fixBranchName, fixVersionInfo); err != nil { + if err = cfp.openFixingPullRequest(fixBranchName, fixDetails); err != nil { return fmt.Errorf("failed while creating a fixing pull request for: %s with version: %s with error: \n%s", - impactedPackage, fixVersionInfo.FixVersion, err.Error()) + fixDetails.ImpactedDependency, fixVersion, err.Error()) } + log.Info(fmt.Sprintf("Created Pull Request updating dependency '%s' to version '%s'", fixDetails.ImpactedDependency, fixDetails.FixVersion)) return } -func (cfp *CreateFixPullRequestsCmd) openFixingPullRequest(impactedPackage, fixBranchName string, fixVersionInfo *utils.FixVersionInfo) (err error) { - log.Info("Checking if there are changes to commit") +func (cfp *CreateFixPullRequestsCmd) openFixingPullRequest(fixBranchName string, fixDetails *utils.FixDetails) (err error) { + log.Debug("Checking if there are changes to commit") isClean, err := cfp.gitManager.IsClean() if err != nil { return } if isClean { - return fmt.Errorf("there were no changes to commit after fixing the package '%s'", impactedPackage) + return fmt.Errorf("there were no changes to commit after fixing the package '%s'", fixDetails.ImpactedDependency) } - commitMessage := cfp.gitManager.GenerateCommitMessage(impactedPackage, fixVersionInfo.FixVersion) - log.Info("Running git add all and commit...") + commitMessage := cfp.gitManager.GenerateCommitMessage(fixDetails.ImpactedDependency, fixDetails.FixVersion) + log.Debug("Running git add all and commit...") if err = cfp.gitManager.AddAllAndCommit(commitMessage); err != nil { return } - log.Info("Pushing branch:", fixBranchName, "...") + log.Debug("Pushing branch:", fixBranchName, "...") if err = cfp.gitManager.Push(false, fixBranchName); err != nil { return } - pullRequestTitle := cfp.gitManager.GeneratePullRequestTitle(impactedPackage, fixVersionInfo.FixVersion) - log.Info("Creating Pull Request form:", fixBranchName, " to:", cfp.details.Branch()) + pullRequestTitle := cfp.gitManager.GeneratePullRequestTitle(fixDetails.ImpactedDependency, fixDetails.FixVersion) + log.Debug("Creating Pull Request form:", fixBranchName, " to:", cfp.details.Branch()) + prBody := commitMessage + "\n\n" + utils.WhatIsFrogbotMd return cfp.details.Client().CreatePullRequest(context.Background(), cfp.details.RepoOwner, cfp.details.RepoName, fixBranchName, cfp.details.Branch(), pullRequestTitle, prBody) } @@ -233,16 +270,17 @@ func (cfp *CreateFixPullRequestsCmd) openFixingPullRequest(impactedPackage, fixB // When aggregate mode is active, there can be only one updated pull request to contain all the available fixes. // In case of an already opened pull request, Frogbot will only update the branch. func (cfp *CreateFixPullRequestsCmd) openAggregatedPullRequest(fixBranchName string) (err error) { - log.Info("Checking if there are changes to commit") + log.Debug("Checking if there are changes to commit") isClean, err := cfp.gitManager.IsClean() if err != nil { return } - if isClean { - return fmt.Errorf("there were no changes in the fix branch '%s'", fixBranchName) + if !isClean { + log.Debug("Couldn't fix any of the impacted dependencies") + return } commitMessage := cfp.gitManager.GenerateAggregatedCommitMessage() - log.Info("Running git add all and commit...") + log.Debug("Running git add all and commit...") if err = cfp.gitManager.AddAllAndCommit(commitMessage); err != nil { return } @@ -250,12 +288,12 @@ func (cfp *CreateFixPullRequestsCmd) openAggregatedPullRequest(fixBranchName str if err != nil { return } - log.Info("Pushing branch:", fixBranchName, "...") + log.Debug("Pushing branch:", fixBranchName, "...") if err = cfp.gitManager.Push(true, fixBranchName); err != nil { return } if !exists { - log.Info("Creating Pull Request form:", fixBranchName, " to:", cfp.details.Branch) + log.Info("Creating Pull Request from:", fixBranchName, "to:", cfp.details.Branch()) prBody := commitMessage + "\n\n" + utils.WhatIsFrogbotMd return cfp.details.Client().CreatePullRequest(context.Background(), cfp.details.RepoOwner, cfp.details.RepoName, fixBranchName, cfp.details.Branch(), utils.AggregatedPullRequestTitleTemplate, prBody) } @@ -263,18 +301,6 @@ func (cfp *CreateFixPullRequestsCmd) openAggregatedPullRequest(fixBranchName str return } -func (cfp *CreateFixPullRequestsCmd) createFixBranch(fixBranchName string, failOnExists bool) (err error) { - exists, err := cfp.gitManager.BranchExistsInRemote(fixBranchName) - if err != nil { - return - } - log.Info("Creating branch", fixBranchName, "...") - if failOnExists && exists { - return fmt.Errorf("branch %s already exists in the remote git repository", fixBranchName) - } - return cfp.gitManager.CreateBranchAndCheckout(fixBranchName) -} - func (cfp *CreateFixPullRequestsCmd) cloneRepository() (tempWd string, restoreDir func() error, err error) { // Create temp working directory tempWd, err = fileutils.CreateTempDir() @@ -294,8 +320,8 @@ func (cfp *CreateFixPullRequestsCmd) cloneRepository() (tempWd string, restoreDi } // Create fixVersionMap - a map with 'impacted package' as key and 'fix version' as value. -func (cfp *CreateFixPullRequestsCmd) createFixVersionsMap(scanResults []services.ScanResponse, isMultipleRoots bool) (map[string]*utils.FixVersionInfo, error) { - fixVersionsMap := map[string]*utils.FixVersionInfo{} +func (cfp *CreateFixPullRequestsCmd) createFixVersionsMap(scanResults []services.ScanResponse, isMultipleRoots bool) (map[string]*utils.FixDetails, error) { + fixVersionsMap := map[string]*utils.FixDetails{} for _, scanResult := range scanResults { if len(scanResult.Vulnerabilities) > 0 { vulnerabilities, err := xrayutils.PrepareVulnerabilities(scanResult.Vulnerabilities, &xrayutils.ExtendedScanResults{}, isMultipleRoots, true) @@ -312,7 +338,7 @@ func (cfp *CreateFixPullRequestsCmd) createFixVersionsMap(scanResults []services return fixVersionsMap, nil } -func (cfp *CreateFixPullRequestsCmd) addVulnerabilityToFixVersionsMap(vulnerability *formats.VulnerabilityOrViolationRow, fixVersionsMap map[string]*utils.FixVersionInfo) error { +func (cfp *CreateFixPullRequestsCmd) addVulnerabilityToFixVersionsMap(vulnerability *formats.VulnerabilityOrViolationRow, fixVersionsMap map[string]*utils.FixDetails) error { if len(vulnerability.FixedVersions) == 0 { return nil } @@ -330,12 +356,13 @@ func (cfp *CreateFixPullRequestsCmd) addVulnerabilityToFixVersionsMap(vulnerabil return err } // First appearance of a version that fixes the current impacted package - fixVersionsMap[vulnerability.ImpactedDependencyName] = utils.NewFixVersionInfo(vulnFixVersion, vulnerability.Technology, isDirectDependency) + fixVersionsMap[vulnerability.ImpactedDependencyName] = utils.NewFixDetails(vulnFixVersion, vulnerability.Technology, isDirectDependency, vulnerability.ImpactedDependencyName) } return nil } -func (cfp *CreateFixPullRequestsCmd) updatePackageToFixedVersion(impactedPackage string, fixVersionInfo *utils.FixVersionInfo) (err error) { +// Updates impacted package, can return ErrUnsupportedFix. +func (cfp *CreateFixPullRequestsCmd) updatePackageToFixedVersion(fixDetails *utils.FixDetails) (err error) { // 'CD' into the relevant working directory if cfp.projectWorkingDir != "" { restoreDir, err := utils.Chdir(cfp.projectWorkingDir) @@ -351,20 +378,29 @@ func (cfp *CreateFixPullRequestsCmd) updatePackageToFixedVersion(impactedPackage } }() } - // Skip build tools dependencies (for example, pip) - // That are not defined in the descriptor file and cannot be fixed by a PR. - if slices.Contains(utils.BuildToolsDependenciesMap[fixVersionInfo.PackageType], impactedPackage) { - log.Info("Skipping vulnerable package", impactedPackage, "since it is not defined in your package descriptor file.", - "Update", impactedPackage, "version to", fixVersionInfo.FixVersion, "to fix this vulnerability.") + if err = isBuildToolsDependency(fixDetails); err != nil { return } if cfp.handlers == nil { cfp.handlers = make(map[coreutils.Technology]packagehandlers.PackageHandler) } - if cfp.handlers[fixVersionInfo.PackageType] == nil { - cfp.handlers[fixVersionInfo.PackageType] = packagehandlers.GetCompatiblePackageHandler(fixVersionInfo, cfp.details) + if cfp.handlers[fixDetails.PackageType] == nil { + cfp.handlers[fixDetails.PackageType] = packagehandlers.GetCompatiblePackageHandler(fixDetails, cfp.details) + } + return cfp.handlers[fixDetails.PackageType].UpdateDependency(fixDetails) +} + +func isBuildToolsDependency(fixDetails *utils.FixDetails) error { + // Skip build tools dependencies (for example, pip) + // that are not defined in the descriptor file and cannot be fixed by a PR. + if slices.Contains(utils.BuildToolsDependenciesMap[fixDetails.PackageType], fixDetails.ImpactedDependency) { + return &utils.ErrUnsupportedFix{ + PackageName: fixDetails.ImpactedDependency, + FixedVersion: fixDetails.FixVersion, + ErrorType: utils.BuildToolsDependencyFixNotSupported, + } } - return cfp.handlers[fixVersionInfo.PackageType].UpdateImpactedPackage(impactedPackage, fixVersionInfo) + return nil } // getMinimalFixVersion find the minimal version that fixes the current impactedPackage; @@ -398,3 +434,11 @@ func parseVersionChangeString(fixVersion string) string { latestVersion = strings.Trim(latestVersion, "]") return latestVersion } + +// During the operation of updating packages, there could be some errors, +// in order to not fail the whole run, we store the errors in strings.builder and log them at the end. +func logAppendedErrorsIfExists(errList strings.Builder) { + if errList.String() != "" { + log.Error("During fixing dependencies operations the following errors occurred:\n", errors.New(errList.String())) + } +} diff --git a/commands/createfixpullrequests_test.go b/commands/createfixpullrequests_test.go index 72310bbde..6ab722484 100644 --- a/commands/createfixpullrequests_test.go +++ b/commands/createfixpullrequests_test.go @@ -1,15 +1,11 @@ package commands import ( - testdatautils "github.com/jfrog/build-info-go/build/testdata" - "github.com/jfrog/frogbot/commands/utils/packagehandlers" "github.com/jfrog/jfrog-client-go/utils/io/fileutils" "github.com/jfrog/jfrog-client-go/utils/log" "github.com/stretchr/testify/assert" "os" "path/filepath" - "regexp" - "strings" "testing" "github.com/jfrog/frogbot/commands/utils" @@ -17,56 +13,6 @@ import ( "github.com/jfrog/jfrog-client-go/xray/services" ) -type FixPackagesTestFunc func(test packageFixTest) CreateFixPullRequestsCmd - -type packageFixTest struct { - technology coreutils.Technology - impactedPackaged string - fixVersion string - packageDescriptor string - testPath string - shouldNotFix bool - fixPackageVersionCmd FixPackagesTestFunc -} - -var packageFixTests = []packageFixTest{ - {technology: coreutils.Npm, impactedPackaged: "minimatch", fixVersion: "3.0.2", packageDescriptor: "package.json", fixPackageVersionCmd: getGenericFixPackageVersionFunc()}, - {technology: coreutils.Go, impactedPackaged: "github.com/google/uuid", fixVersion: "1.3.0", packageDescriptor: "go.mod", fixPackageVersionCmd: getGenericFixPackageVersionFunc()}, - {technology: coreutils.Go, impactedPackaged: "github.com/golang/go", fixVersion: "1.20.3", packageDescriptor: "go.mod", fixPackageVersionCmd: getGenericFixPackageVersionFunc(), shouldNotFix: true}, - {technology: coreutils.Yarn, impactedPackaged: "minimist", fixVersion: "1.2.6", packageDescriptor: "package.json", fixPackageVersionCmd: getGenericFixPackageVersionFunc()}, - {technology: coreutils.Pipenv, impactedPackaged: "pyjwt", fixVersion: "2.4.0", packageDescriptor: "Pipfile", fixPackageVersionCmd: getGenericFixPackageVersionFunc()}, - {technology: coreutils.Pipenv, impactedPackaged: "Pyjwt", fixVersion: "2.4.0", packageDescriptor: "Pipfile", fixPackageVersionCmd: getGenericFixPackageVersionFunc()}, - {technology: coreutils.Poetry, impactedPackaged: "pyjwt", fixVersion: "2.4.0", packageDescriptor: "pyproject.toml", fixPackageVersionCmd: getGenericFixPackageVersionFunc()}, - {technology: coreutils.Poetry, impactedPackaged: "Pyjwt", fixVersion: "2.4.0", packageDescriptor: "pyproject.toml", fixPackageVersionCmd: getGenericFixPackageVersionFunc()}, - {technology: coreutils.Pip, impactedPackaged: "pyjwt", fixVersion: "2.4.0", packageDescriptor: "requirements.txt", fixPackageVersionCmd: getGenericFixPackageVersionFunc()}, - {technology: coreutils.Pip, impactedPackaged: "PyJwt", fixVersion: "2.4.0", packageDescriptor: "requirements.txt", fixPackageVersionCmd: getGenericFixPackageVersionFunc()}, - {technology: coreutils.Pip, impactedPackaged: "pyjwt", fixVersion: "2.4.0", packageDescriptor: "setup.py", fixPackageVersionCmd: getGenericFixPackageVersionFunc()}, - {technology: coreutils.Pip, impactedPackaged: "pip", fixVersion: "23.1", packageDescriptor: "setup.py", fixPackageVersionCmd: getGenericFixPackageVersionFunc(), shouldNotFix: true}, - {technology: coreutils.Pip, impactedPackaged: "wheel", fixVersion: "2.3.0", packageDescriptor: "setup.py", fixPackageVersionCmd: getGenericFixPackageVersionFunc(), shouldNotFix: true}, - {technology: coreutils.Pip, impactedPackaged: "setuptools", fixVersion: "66.6.6", packageDescriptor: "setup.py", fixPackageVersionCmd: getGenericFixPackageVersionFunc(), shouldNotFix: true}, -} - -var requirementsFile = "oslo.config>=1.12.1,<1.13\noslo.utils<5.0,>=4.0.0\nparamiko==2.7.2\npasslib<=1.7.4\nprance>=0.9.0\nprompt-toolkit~=1.0.15\npyinotify>0.9.6\nPyJWT>1.7.1\nurllib3 > 1.1.9, < 1.5.*" - -type pipPackageRegexTest struct { - packageName string - expectedRequirement string -} - -var pipPackagesRegexTests = []pipPackageRegexTest{ - {"oslo.config", "oslo.config>=1.12.1,<1.13"}, - {"oslo.utils", "oslo.utils<5.0,>=4.0.0"}, - {"paramiko", "paramiko==2.7.2"}, - {"passlib", "passlib<=1.7.4"}, - {"PassLib", "passlib<=1.7.4"}, - {"prance", "prance>=0.9.0"}, - {"prompt-toolkit", "prompt-toolkit~=1.0.15"}, - {"pyinotify", "pyinotify>0.9.6"}, - {"pyjwt", "pyjwt>1.7.1"}, - {"PyJWT", "pyjwt>1.7.1"}, - {"urllib3", "urllib3 > 1.1.9, < 1.5.*"}, -} - var testPackagesData = []struct { packageType coreutils.Technology commandName string @@ -107,60 +53,6 @@ var testPackagesData = []struct { }, } -func getGenericFixPackageVersionFunc() FixPackagesTestFunc { - return func(test packageFixTest) CreateFixPullRequestsCmd { - return CreateFixPullRequestsCmd{ - details: &utils.ScanDetails{ - Project: &utils.Project{ - PipRequirementsFile: test.packageDescriptor, - WorkingDirs: []string{test.testPath}, - }, - }, - } - } -} - -func TestGenericFixPackageVersion(t *testing.T) { - currentDir, testdataDir := getTestDataDir(t) - defer func() { - assert.NoError(t, os.Chdir(currentDir)) - }() - - for _, test := range packageFixTests { - func() { - // Create a temp technology project - projectPath := filepath.Join(testdataDir, test.technology.ToString()) - tmpProjectPath, cleanup := testdatautils.CreateTestProject(t, projectPath) - defer cleanup() - test.testPath = tmpProjectPath - assert.NoError(t, os.Chdir(tmpProjectPath)) - - t.Run(test.technology.ToString(), func(t *testing.T) { - cfg := test.fixPackageVersionCmd(test) - // Fix impacted package for each technology - fixVersionInfo := utils.NewFixVersionInfo(test.fixVersion, test.technology, true) - assert.NoError(t, cfg.updatePackageToFixedVersion(test.impactedPackaged, fixVersionInfo)) - file, err := os.ReadFile(test.packageDescriptor) - assert.NoError(t, err) - if test.shouldNotFix { - assert.NotContains(t, string(file), test.fixVersion) - } else { - assert.Contains(t, string(file), test.fixVersion) - // Verify that case-sensitive packages in python are lowered - assert.Contains(t, string(file), strings.ToLower(test.impactedPackaged)) - } - }) - }() - } -} -func getTestDataDir(t *testing.T) (string, string) { - currentDir, err := os.Getwd() - assert.NoError(t, err) - testdataDir, err := filepath.Abs("testdata/projects") - assert.NoError(t, err) - return currentDir, testdataDir -} - // / 1.0 --> 1.0 ≤ x // / (,1.0] --> x ≤ 1.0 // / (,1.0) --> x < 1.0 @@ -211,14 +103,6 @@ func TestGenerateFixBranchName(t *testing.T) { } } -func TestPipPackageRegex(t *testing.T) { - for _, pack := range pipPackagesRegexTests { - re := regexp.MustCompile(packagehandlers.PythonPackageRegexPrefix + "(" + pack.packageName + "|" + strings.ToLower(pack.packageName) + ")" + packagehandlers.PythonPackageRegexSuffix) - found := re.FindString(requirementsFile) - assert.Equal(t, pack.expectedRequirement, strings.ToLower(found)) - } -} - func TestPackageTypeFromScan(t *testing.T) { environmentVars, restoreEnv := verifyEnv(t) defer restoreEnv() @@ -285,6 +169,20 @@ func TestGetMinimalFixVersion(t *testing.T) { } } +// Verifies unsupported packages return specific error +// Other logic is implemented inside each package-handler. +func TestUpdatePackageToFixedVersion(t *testing.T) { + var testScan CreateFixPullRequestsCmd + for tech, buildToolsDependencies := range utils.BuildToolsDependenciesMap { + for _, impactedDependency := range buildToolsDependencies { + fixDetails := &utils.FixDetails{FixVersion: "3.3.3", PackageType: tech, ImpactedDependency: impactedDependency, DirectDependency: true} + err := testScan.updatePackageToFixedVersion(fixDetails) + assert.Error(t, err, "Expected error to occur") + assert.IsType(t, &utils.ErrUnsupportedFix{}, err, "Expected unsupported fix error") + } + } +} + func verifyTechnologyNaming(t *testing.T, scanResponse []services.ScanResponse, expectedType coreutils.Technology) { for _, resp := range scanResponse { for _, vulnerability := range resp.Vulnerabilities { diff --git a/commands/scanandfixrepos.go b/commands/scanandfixrepos.go index a6f8a665f..c7a13907a 100644 --- a/commands/scanandfixrepos.go +++ b/commands/scanandfixrepos.go @@ -22,12 +22,10 @@ type ScanAndFixRepositories struct { func (saf *ScanAndFixRepositories) Run(configAggregator utils.FrogbotConfigAggregator, client vcsclient.VcsClient) error { var errList strings.Builder for repoNum := range configAggregator { - err := saf.scanAndFixSingleRepository(&configAggregator[repoNum], client) - if err != nil { + if err := saf.scanAndFixSingleRepository(&configAggregator[repoNum], client); err != nil { errList.WriteString(fmt.Sprintf("repository %s returned the following error: \n%s\n", configAggregator[repoNum].RepoName, err.Error())) } } - if errList.String() != "" { return errors.New(errList.String()) } @@ -45,13 +43,17 @@ func (saf *ScanAndFixRepositories) scanAndFixSingleRepository(repoConfig *utils. continue } if err = saf.downloadAndRunScanAndFix(repoConfig, branch, client); err != nil { - // Scan failed, mark commit status failed with error info - e := saf.setCommitBuildStatus(client, repoConfig, vcsclient.Fail, checkedCommit, fmt.Sprintf("Frogbot error: %s", err)) - if e != nil { - return errors.New(err.Error() + "\n" + e.Error()) + if _, isCustomError := err.(*utils.ErrUnsupportedFix); isCustomError { + log.Debug(err.Error()) + } else { + // Scan failed, mark commit status failed with error info + if e := saf.setCommitBuildStatus(client, repoConfig, vcsclient.Fail, checkedCommit, fmt.Sprintf("Frogbot error: %s", err)); e != nil { + return errors.Join(e, err) + } + return err } - return err } + // Mark successful run if err = saf.setCommitBuildStatus(client, repoConfig, vcsclient.Pass, checkedCommit, utils.CommitStatusDescription); err != nil { return err } diff --git a/commands/testdata/indirect-projects/go/go.mod b/commands/testdata/indirect-projects/go/go.mod new file mode 100644 index 000000000..eac5f66ae --- /dev/null +++ b/commands/testdata/indirect-projects/go/go.mod @@ -0,0 +1,22 @@ +module goproject + +go 1.19 + +require github.com/gin-gonic/gin v1.7.3 + +require ( + github.com/gin-contrib/sse v0.1.0 // indirect + github.com/go-playground/locales v0.13.0 // indirect + github.com/go-playground/universal-translator v0.17.0 // indirect + github.com/go-playground/validator/v10 v10.4.1 // indirect + github.com/golang/protobuf v1.3.3 // indirect + github.com/json-iterator/go v1.1.9 // indirect + github.com/leodido/go-urn v1.2.0 // indirect + github.com/mattn/go-isatty v0.0.12 // indirect + github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421 // indirect + github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742 // indirect + github.com/ugorji/go/codec v1.1.7 // indirect + golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 // indirect + golang.org/x/sys v0.0.0-20200116001909-b77594299b42 // indirect + gopkg.in/yaml.v2 v2.2.8 // indirect +) diff --git a/commands/testdata/indirect-projects/go/main.go b/commands/testdata/indirect-projects/go/main.go new file mode 100644 index 000000000..34a990c92 --- /dev/null +++ b/commands/testdata/indirect-projects/go/main.go @@ -0,0 +1,10 @@ +package main + +import ( + "github.com/gin-gonic/gin" +) + +func main() { + print("test") + _ = gin.Default() +} diff --git a/commands/testdata/indirect-projects/maven/pom.xml b/commands/testdata/indirect-projects/maven/pom.xml new file mode 100644 index 000000000..ba270fc08 --- /dev/null +++ b/commands/testdata/indirect-projects/maven/pom.xml @@ -0,0 +1,67 @@ + + + 4.0.0 + org.jfrog.test + multi + 3.7-SNAPSHOT + pom + Simple Multi Modules Build + + + UTF-8 + 1.8 + 1.8 + + + + + junit + junit + 3.8.1 + test + + + + + + + + org.apache.maven.plugins + maven-jar-plugin + 2.4 + + + org.apache.maven.plugins + maven-war-plugin + 2.4 + + + org.apache.maven.plugins + maven-source-plugin + 2.1.2 + + + + + + org.apache.maven.plugins + maven-jar-plugin + + + false + + + + + org.apache.maven.plugins + maven-war-plugin + + + false + + + + + + diff --git a/commands/testdata/indirect-projects/pip/requirements.txt b/commands/testdata/indirect-projects/pip/requirements.txt new file mode 100644 index 000000000..802474953 --- /dev/null +++ b/commands/testdata/indirect-projects/pip/requirements.txt @@ -0,0 +1 @@ +requests==2.21.0 \ No newline at end of file diff --git a/commands/testdata/indirect-projects/pip/setup.py b/commands/testdata/indirect-projects/pip/setup.py new file mode 100644 index 000000000..3e10635fd --- /dev/null +++ b/commands/testdata/indirect-projects/pip/setup.py @@ -0,0 +1,9 @@ +from setuptools import setup, find_packages + +setup( + name="pip-example", + version="1.2.3", + install_requires=[ + "requests==2.21.0" + ], +) diff --git a/commands/testdata/indirect-projects/pipenv/Pipfile b/commands/testdata/indirect-projects/pipenv/Pipfile new file mode 100644 index 000000000..6d367636b --- /dev/null +++ b/commands/testdata/indirect-projects/pipenv/Pipfile @@ -0,0 +1,12 @@ +[[source]] +url = "https://pypi.python.org/simple" +verify_ssl = true +name = "pypi" + +[packages] +requests = "==2.21.0" + +[dev-packages] + +[requires] +python_version = "*" diff --git a/commands/testdata/indirect-projects/pipenv/Pipfile.lock b/commands/testdata/indirect-projects/pipenv/Pipfile.lock new file mode 100644 index 000000000..120702377 --- /dev/null +++ b/commands/testdata/indirect-projects/pipenv/Pipfile.lock @@ -0,0 +1,59 @@ +{ + "_meta": { + "hash": { + "sha256": "558fa3d8925a85d24792479c399941b74eda5d07bde65eace645285c07547361" + }, + "pipfile-spec": 6, + "requires": { + "python_version": "*" + }, + "sources": [ + { + "name": "pypi", + "url": "https://pypi.python.org/simple", + "verify_ssl": true + } + ] + }, + "default": { + "certifi": { + "hashes": [ + "sha256:35824b4c3a97115964b408844d64aa14db1cc518f6562e8d7261699d1350a9e3", + "sha256:4ad3232f5e926d6718ec31cfc1fcadfde020920e278684144551c91769c7bc18" + ], + "markers": "python_version >= '3.6'", + "version": "==2022.12.7" + }, + "chardet": { + "hashes": [ + "sha256:84ab92ed1c4d4f16916e05906b6b75a6c0fb5db821cc65e70cbd64a3e2a5eaae", + "sha256:fc323ffcaeaed0e0a02bf4d117757b98aed530d9ed4531e3e15460124c106691" + ], + "version": "==3.0.4" + }, + "idna": { + "hashes": [ + "sha256:c357b3f628cf53ae2c4c05627ecc484553142ca23264e593d327bcde5e9c3407", + "sha256:ea8b7f6188e6fa117537c3df7da9fc686d485087abf6ac197f9c46432f7e4a3c" + ], + "version": "==2.8" + }, + "requests": { + "hashes": [ + "sha256:502a824f31acdacb3a35b6690b5fbf0bc41d63a24a45c4004352b0242707598e", + "sha256:7bf2a778576d825600030a110f3c0e3e8edc51dfaafe1c146e39a2027784957b" + ], + "index": "pypi", + "version": "==2.21.0" + }, + "urllib3": { + "hashes": [ + "sha256:2393a695cd12afedd0dcb26fe5d50d0cf248e5a66f75dbd89a3d4eb333a61af4", + "sha256:a637e5fae88995b256e3409dc4d52c2e2e0ba32c42a6365fee8bbd2238de3cfb" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3' and python_version < '4'", + "version": "==1.24.3" + } + }, + "develop": {} +} diff --git a/commands/testdata/indirect-projects/poetry/pyproject.toml b/commands/testdata/indirect-projects/poetry/pyproject.toml new file mode 100644 index 000000000..4696f1574 --- /dev/null +++ b/commands/testdata/indirect-projects/poetry/pyproject.toml @@ -0,0 +1,13 @@ +[tool.poetry] +name = "poetry-project" +version = "0.1.0" +description = "" +authors = ["Your Name "] + +[tool.poetry.dependencies] +python = "^3.10" +requests = "==2.21.0" + +[build-system] +requires = ["poetry-core>=1.0.0"] +build-backend = "poetry.core.masonry.api" diff --git a/commands/utils/consts.go b/commands/utils/consts.go index dc50f4a67..15d6cb87a 100644 --- a/commands/utils/consts.go +++ b/commands/utils/consts.go @@ -105,3 +105,10 @@ const ( frogbotAuthorName = "JFrog-Frogbot" frogbotAuthorEmail = "eco-system+frogbot@jfrog.com" ) + +type UnsupportedErrorType string + +const ( + IndirectDependencyFixNotSupported UnsupportedErrorType = "IndirectDependencyFixNotSupported" + BuildToolsDependencyFixNotSupported UnsupportedErrorType = "BuildToolsDependencyFixNotSupported" +) diff --git a/commands/utils/git.go b/commands/utils/git.go index 754ab548d..ac1ccdbe8 100644 --- a/commands/utils/git.go +++ b/commands/utils/git.go @@ -97,7 +97,7 @@ func (gm *GitManager) Clone(destinationPath, branchName string) error { log.Debug("Since no branch name was set, assuming 'master' as the default branch") branchName = "master" } - log.Info(fmt.Sprintf("Cloning repository with these details:\nClone url: %s remote name: %s, branch: %s", repoURL, gm.remoteName, getFullBranchName(branchName))) + log.Debug(fmt.Sprintf("Cloning repository with these details:\nClone url: %s remote name: %s, branch: %s", repoURL, gm.remoteName, getFullBranchName(branchName))) cloneOptions := &git.CloneOptions{ URL: repoURL, Auth: gm.auth, @@ -301,7 +301,7 @@ func (gm *GitManager) GeneratePullRequestTitle(impactedPackage string, version s } // Generates unique branch name constructed by all the vulnerable packages. -func (gm *GitManager) GenerateAggregatedFixBranchName(versionsMap map[string]*FixVersionInfo) (fixBranchName string, err error) { +func (gm *GitManager) GenerateAggregatedFixBranchName(versionsMap map[string]*FixDetails) (fixBranchName string, err error) { hash, err := fixVersionsMapToMd5Hash(versionsMap) if err != nil { return diff --git a/commands/utils/git_test.go b/commands/utils/git_test.go index 8345def00..df2f66a41 100644 --- a/commands/utils/git_test.go +++ b/commands/utils/git_test.go @@ -10,25 +10,25 @@ func TestGitManager_GenerateCommitMessage(t *testing.T) { tests := []struct { gitManager GitManager impactedPackage string - fixVersion FixVersionInfo + fixVersion FixDetails expected string description string }{ { gitManager: GitManager{customTemplates: CustomTemplates{commitMessageTemplate: ": bump ${IMPACTED_PACKAGE}"}}, impactedPackage: "mquery", - fixVersion: FixVersionInfo{FixVersion: "3.4.5"}, + fixVersion: FixDetails{FixVersion: "3.4.5"}, expected: ": bump mquery", description: "Custom prefix", }, { gitManager: GitManager{customTemplates: CustomTemplates{commitMessageTemplate: "[scope]: Upgrade package ${IMPACTED_PACKAGE} to ${FIX_VERSION}"}}, - impactedPackage: "mquery", fixVersion: FixVersionInfo{FixVersion: "3.4.5"}, + impactedPackage: "mquery", fixVersion: FixDetails{FixVersion: "3.4.5"}, expected: "[scope]: Upgrade package mquery to 3.4.5", description: "Default template", }, { gitManager: GitManager{customTemplates: CustomTemplates{commitMessageTemplate: ""}}, - impactedPackage: "mquery", fixVersion: FixVersionInfo{FixVersion: "3.4.5"}, + impactedPackage: "mquery", fixVersion: FixDetails{FixVersion: "3.4.5"}, expected: "Upgrade mquery to 3.4.5", description: "Default template", }, @@ -45,27 +45,27 @@ func TestGitManager_GenerateFixBranchName(t *testing.T) { tests := []struct { gitManager GitManager impactedPackage string - fixVersion FixVersionInfo + fixVersion FixDetails expected string description string }{ { gitManager: GitManager{customTemplates: CustomTemplates{branchNameTemplate: "[Feature]-${IMPACTED_PACKAGE}-${BRANCH_NAME_HASH}"}}, impactedPackage: "mquery", - fixVersion: FixVersionInfo{FixVersion: "3.4.5"}, + fixVersion: FixDetails{FixVersion: "3.4.5"}, expected: "[Feature]-mquery-41b1f45136b25e3624b15999bd57a476", description: "Custom template", }, { gitManager: GitManager{customTemplates: CustomTemplates{branchNameTemplate: ""}}, impactedPackage: "mquery", - fixVersion: FixVersionInfo{FixVersion: "3.4.5"}, + fixVersion: FixDetails{FixVersion: "3.4.5"}, expected: "frogbot-mquery-41b1f45136b25e3624b15999bd57a476", description: "No template", }, { gitManager: GitManager{customTemplates: CustomTemplates{branchNameTemplate: "just-a-branch-${BRANCH_NAME_HASH}"}}, impactedPackage: "mquery", - fixVersion: FixVersionInfo{FixVersion: "3.4.5"}, + fixVersion: FixDetails{FixVersion: "3.4.5"}, expected: "just-a-branch-41b1f45136b25e3624b15999bd57a476", description: "Custom template without inputs", }, @@ -83,28 +83,28 @@ func TestGitManager_GeneratePullRequestTitle(t *testing.T) { tests := []struct { gitManager GitManager impactedPackage string - fixVersion FixVersionInfo + fixVersion FixDetails expected string description string }{ { gitManager: GitManager{customTemplates: CustomTemplates{pullRequestTitleTemplate: "[CustomPR] update ${IMPACTED_PACKAGE} to ${FIX_VERSION}"}}, impactedPackage: "mquery", - fixVersion: FixVersionInfo{FixVersion: "3.4.5"}, + fixVersion: FixDetails{FixVersion: "3.4.5"}, expected: "[CustomPR] update mquery to 3.4.5", description: "Custom template", }, { gitManager: GitManager{customTemplates: CustomTemplates{pullRequestTitleTemplate: "[CustomPR] update ${IMPACTED_PACKAGE}"}}, impactedPackage: "mquery", - fixVersion: FixVersionInfo{FixVersion: "3.4.5"}, + fixVersion: FixDetails{FixVersion: "3.4.5"}, expected: "[CustomPR] update mquery", description: "Custom template one var", }, { gitManager: GitManager{customTemplates: CustomTemplates{pullRequestTitleTemplate: ""}}, impactedPackage: "mquery", - fixVersion: FixVersionInfo{FixVersion: "3.4.5"}, + fixVersion: FixDetails{FixVersion: "3.4.5"}, expected: "[🐸 Frogbot] Update version of mquery to 3.4.5", description: "No prefix", }, @@ -119,28 +119,28 @@ func TestGitManager_GeneratePullRequestTitle(t *testing.T) { func TestGitManager_GenerateAggregatedFixBranchName(t *testing.T) { tests := []struct { - fixVersionMapFirst map[string]*FixVersionInfo - fixVersionMapSecond map[string]*FixVersionInfo + fixVersionMapFirst map[string]*FixDetails + fixVersionMapSecond map[string]*FixDetails gitManager GitManager equal bool desc string }{ { - fixVersionMapFirst: map[string]*FixVersionInfo{ + fixVersionMapFirst: map[string]*FixDetails{ "pkg": {FixVersion: "1.2.3", PackageType: coreutils.Npm, DirectDependency: false}, "pkg2": {FixVersion: "1.5.3", PackageType: coreutils.Npm, DirectDependency: false}}, - fixVersionMapSecond: map[string]*FixVersionInfo{ + fixVersionMapSecond: map[string]*FixDetails{ "pkg": {FixVersion: "1.2.3", PackageType: coreutils.Npm, DirectDependency: false}, "pkg2": {FixVersion: "1.5.3", PackageType: coreutils.Npm, DirectDependency: false}}, equal: true, desc: "should be equal", gitManager: GitManager{}, }, { - fixVersionMapFirst: map[string]*FixVersionInfo{ + fixVersionMapFirst: map[string]*FixDetails{ "pkg": {FixVersion: "1.2.3", PackageType: coreutils.Npm, DirectDependency: false}, "pkg2": {FixVersion: "1.5.3", PackageType: coreutils.Npm, DirectDependency: false}, }, - fixVersionMapSecond: map[string]*FixVersionInfo{ + fixVersionMapSecond: map[string]*FixDetails{ "pkgOther": {FixVersion: "1.2.3", PackageType: coreutils.Npm, DirectDependency: false}, "pkg2": {FixVersion: "1.5.3", PackageType: coreutils.Npm, DirectDependency: false}}, equal: false, @@ -148,11 +148,11 @@ func TestGitManager_GenerateAggregatedFixBranchName(t *testing.T) { gitManager: GitManager{}, }, { - fixVersionMapFirst: map[string]*FixVersionInfo{ + fixVersionMapFirst: map[string]*FixDetails{ "pkg": {FixVersion: "1.2.3", PackageType: coreutils.Npm, DirectDependency: false}, "pkg2": {FixVersion: "1.5.3", PackageType: coreutils.Npm, DirectDependency: false}, }, - fixVersionMapSecond: map[string]*FixVersionInfo{ + fixVersionMapSecond: map[string]*FixDetails{ "pkgOther": {FixVersion: "1.2.3", PackageType: coreutils.Npm, DirectDependency: false}, "pkg2": {FixVersion: "1.5.3", PackageType: coreutils.Npm, DirectDependency: false}}, equal: true, diff --git a/commands/utils/packagehandlers/commonPackageHandler.go b/commands/utils/packagehandlers/commonPackageHandler.go new file mode 100644 index 000000000..6d2a35e6f --- /dev/null +++ b/commands/utils/packagehandlers/commonPackageHandler.go @@ -0,0 +1,59 @@ +package packagehandlers + +import ( + "fmt" + "github.com/jfrog/frogbot/commands/utils" + "github.com/jfrog/jfrog-cli-core/v2/utils/coreutils" + "github.com/jfrog/jfrog-client-go/utils/log" + "os/exec" + "strings" +) + +// PackageHandler interface to hold operations on packages +type PackageHandler interface { + UpdateDependency(details *utils.FixDetails) error +} + +func GetCompatiblePackageHandler(fixVersionInfo *utils.FixDetails, details *utils.ScanDetails) (handler PackageHandler) { + switch fixVersionInfo.PackageType { + case coreutils.Go: + handler = &GoPackageHandler{} + case coreutils.Poetry: + handler = &PythonPackageHandler{} + case coreutils.Pipenv: + handler = &PythonPackageHandler{} + case coreutils.Npm: + handler = &NpmPackageHandler{} + case coreutils.Yarn: + handler = &YarnPackageHandler{} + case coreutils.Pip: + handler = &PythonPackageHandler{pipRequirementsFile: details.PipRequirementsFile} + case coreutils.Maven: + handler = &MavenPackageHandler{depsRepo: details.Repository, ServerDetails: details.ServerDetails} + } + return +} + +type CommonPackageHandler struct{} + +// UpdateDependency updates the impacted package to the fixed version +func (cph *CommonPackageHandler) UpdateDependency(fixDetails *utils.FixDetails, extraArgs ...string) (err error) { + // Lower the package name to avoid duplicates + impactedPackage := strings.ToLower(fixDetails.ImpactedDependency) + commandArgs := []string{fixDetails.PackageType.GetPackageInstallOperator()} + commandArgs = append(commandArgs, extraArgs...) + operator := fixDetails.PackageType.GetPackageOperator() + fixedPackage := impactedPackage + operator + fixDetails.FixVersion + commandArgs = append(commandArgs, fixedPackage) + return runPackageMangerCommand(fixDetails.PackageType.GetExecCommandName(), commandArgs) +} + +func runPackageMangerCommand(commandName string, commandArgs []string) error { + fullCommand := commandName + " " + strings.Join(commandArgs, " ") + log.Debug(fmt.Sprintf("Running '%s'", fullCommand)) + output, err := exec.Command(commandName, commandArgs...).CombinedOutput() // #nosec G204 + if err != nil { + return fmt.Errorf("%s command failed: %s\n%s", fullCommand, err.Error(), output) + } + return nil +} diff --git a/commands/utils/packagehandlers/genericpackagehandler.go b/commands/utils/packagehandlers/genericpackagehandler.go deleted file mode 100644 index 4a9017669..000000000 --- a/commands/utils/packagehandlers/genericpackagehandler.go +++ /dev/null @@ -1,58 +0,0 @@ -package packagehandlers - -import ( - "fmt" - "github.com/jfrog/frogbot/commands/utils" - "github.com/jfrog/jfrog-cli-core/v2/utils/coreutils" - "github.com/jfrog/jfrog-client-go/utils/log" - "os/exec" - "strings" -) - -// PackageHandler interface to hold operations on packages -type PackageHandler interface { - UpdateImpactedPackage(impactedPackage string, fixVersionInfo *utils.FixVersionInfo, extraArgs ...string) error -} - -func GetCompatiblePackageHandler(fixVersionInfo *utils.FixVersionInfo, details *utils.ScanDetails) PackageHandler { - switch fixVersionInfo.PackageType { - case coreutils.Maven: - return &MavenPackageHandler{depsRepo: details.Repository, ServerDetails: details.ServerDetails} - case coreutils.Poetry: - return &PythonPackageHandler{} - case coreutils.Pip: - return &PythonPackageHandler{pipRequirementsFile: details.PipRequirementsFile} - default: - return &GenericPackageHandler{FixVersionInfo: fixVersionInfo} - } -} - -type GenericPackageHandler struct { - FixVersionInfo *utils.FixVersionInfo -} - -// UpdateImpactedPackage updates the impacted package to the fixed version -func (g *GenericPackageHandler) UpdateImpactedPackage(impactedPackage string, fixVersionInfo *utils.FixVersionInfo, extraArgs ...string) error { - // Indirect package fix should we implemented for each package handler - if !fixVersionInfo.DirectDependency { - return &utils.ErrUnsupportedIndirectFix{PackageName: impactedPackage, FixedVersion: fixVersionInfo.FixVersion} - } - // Lower the package name to avoid duplicates - impactedPackage = strings.ToLower(impactedPackage) - commandArgs := []string{fixVersionInfo.PackageType.GetPackageInstallOperator()} - commandArgs = append(commandArgs, extraArgs...) - operator := fixVersionInfo.PackageType.GetPackageOperator() - fixedPackage := impactedPackage + operator + fixVersionInfo.FixVersion - commandArgs = append(commandArgs, fixedPackage) - return runPackageMangerCommand(fixVersionInfo.PackageType.GetExecCommandName(), commandArgs) -} - -func runPackageMangerCommand(commandName string, commandArgs []string) error { - fullCommand := commandName + " " + strings.Join(commandArgs, " ") - log.Debug(fmt.Sprintf("Running '%s'", fullCommand)) - output, err := exec.Command(commandName, commandArgs...).CombinedOutput() // #nosec G204 - if err != nil { - return fmt.Errorf("%s command failed: %s\n%s", fullCommand, err.Error(), output) - } - return nil -} diff --git a/commands/utils/packagehandlers/gopackagehandler.go b/commands/utils/packagehandlers/gopackagehandler.go new file mode 100644 index 000000000..27c075eb9 --- /dev/null +++ b/commands/utils/packagehandlers/gopackagehandler.go @@ -0,0 +1,14 @@ +package packagehandlers + +import ( + "github.com/jfrog/frogbot/commands/utils" +) + +type GoPackageHandler struct { + CommonPackageHandler +} + +func (golang *GoPackageHandler) UpdateDependency(fixDetails *utils.FixDetails) error { + // In Golang, we can address every dependency as a direct dependency. + return golang.CommonPackageHandler.UpdateDependency(fixDetails) +} diff --git a/commands/utils/packagehandlers/mavenpackagehandler.go b/commands/utils/packagehandlers/mavenpackagehandler.go index f0565383a..7b59e2a80 100644 --- a/commands/utils/packagehandlers/mavenpackagehandler.go +++ b/commands/utils/packagehandlers/mavenpackagehandler.go @@ -143,7 +143,7 @@ type pomDependencyDetails struct { } type MavenPackageHandler struct { - GenericPackageHandler + CommonPackageHandler // mavenDepToPropertyMap holds a map of direct dependencies found in pom.xml. mavenDepToPropertyMap map[string]pomDependencyDetails // pomPaths holds the paths to all the pom.xml files that are related to the current project. @@ -156,7 +156,7 @@ type MavenPackageHandler struct { depsRepo string } -func (mph *MavenPackageHandler) UpdateImpactedPackage(impactedPackage string, fixVersionInfo *utils.FixVersionInfo, _ ...string) error { +func (mph *MavenPackageHandler) UpdateDependency(fixDetails *utils.FixDetails) error { if err := mph.installMavenGavReader(); err != nil { return err } @@ -176,15 +176,19 @@ func (mph *MavenPackageHandler) UpdateImpactedPackage(impactedPackage string, fi var depDetails pomDependencyDetails var exists bool // Check if the impacted package is a direct dependency - if depDetails, exists = mph.mavenDepToPropertyMap[impactedPackage]; !exists { - return fmt.Errorf(utils.SkipIndirectVulnerabilitiesMsg, impactedPackage, fixVersionInfo.FixVersion) + impactedDependency := fixDetails.ImpactedDependency + if depDetails, exists = mph.mavenDepToPropertyMap[impactedDependency]; !exists { + return &utils.ErrUnsupportedFix{ + PackageName: fixDetails.ImpactedDependency, + FixedVersion: fixDetails.FixVersion, + ErrorType: utils.IndirectDependencyFixNotSupported, + } } - if len(depDetails.properties) > 0 { - return mph.updateProperties(&depDetails, fixVersionInfo.FixVersion) + return mph.updateProperties(&depDetails, fixDetails.FixVersion) } - return mph.updatePackageVersion(impactedPackage, fixVersionInfo.FixVersion, depDetails.foundInDependencyManagement) + return mph.updatePackageVersion(fixDetails.ImpactedDependency, fixDetails.FixVersion, depDetails.foundInDependencyManagement) } func (mph *MavenPackageHandler) installMavenGavReader() (err error) { diff --git a/commands/utils/packagehandlers/npmpackagehandler.go b/commands/utils/packagehandlers/npmpackagehandler.go new file mode 100644 index 000000000..624ac80b3 --- /dev/null +++ b/commands/utils/packagehandlers/npmpackagehandler.go @@ -0,0 +1,23 @@ +package packagehandlers + +import "github.com/jfrog/frogbot/commands/utils" + +type NpmPackageHandler struct { + CommonPackageHandler +} + +func (npm *NpmPackageHandler) UpdateDependency(fixDetails *utils.FixDetails) error { + if fixDetails.DirectDependency { + return npm.updateDirectDependency(fixDetails) + } else { + return &utils.ErrUnsupportedFix{ + PackageName: fixDetails.ImpactedDependency, + FixedVersion: fixDetails.FixVersion, + ErrorType: utils.IndirectDependencyFixNotSupported, + } + } +} + +func (npm *NpmPackageHandler) updateDirectDependency(fixDetails *utils.FixDetails, extraArgs ...string) (err error) { + return npm.CommonPackageHandler.UpdateDependency(fixDetails, extraArgs...) +} diff --git a/commands/utils/packagehandlers/packagehandlers_test.go b/commands/utils/packagehandlers/packagehandlers_test.go index 2c4f948c7..ef7253d6b 100644 --- a/commands/utils/packagehandlers/packagehandlers_test.go +++ b/commands/utils/packagehandlers/packagehandlers_test.go @@ -2,34 +2,252 @@ package packagehandlers import ( "fmt" + testdatautils "github.com/jfrog/build-info-go/build/testdata" "github.com/jfrog/frogbot/commands/utils" + "github.com/jfrog/jfrog-cli-core/v2/utils/coreutils" "github.com/jfrog/jfrog-client-go/utils/io/fileutils" "github.com/stretchr/testify/assert" "os" "path/filepath" + "regexp" + "strconv" + "strings" "testing" ) -func TestFixVersionInfo_UpdateFixVersion(t *testing.T) { - type testCase struct { - fixVersionInfo utils.FixVersionInfo - newFixVersion string - expectedOutput string +type dependencyFixTest struct { + fixVersionInfo *utils.FixDetails + fixSupported bool +} + +type pythonIndirectDependencies struct { + dependencyFixTest + requirementsPath string +} + +const requirementsFile = "oslo.config>=1.12.1,<1.13\noslo.utils<5.0,>=4.0.0\nparamiko==2.7.2\npasslib<=1.7.4\nprance>=0.9.0\nprompt-toolkit~=1.0.15\npyinotify>0.9.6\nPyJWT>1.7.1\nurllib3 > 1.1.9, < 1.5.*" + +type pipPackageRegexTest struct { + packageName string + expectedRequirement string +} + +// Go +func TestGoPackageHandler_UpdateDependency(t *testing.T) { + goPackageHandler := GoPackageHandler{} + testcases := []dependencyFixTest{ + { + fixVersionInfo: &utils.FixDetails{ + FixVersion: "0.0.0-20201216223049-8b5274cf687f", + PackageType: coreutils.Go, + DirectDependency: false, + ImpactedDependency: "golang.org/x/crypto", + }, fixSupported: true, + }, + { + fixVersionInfo: &utils.FixDetails{ + FixVersion: "1.7.7", + PackageType: coreutils.Go, + DirectDependency: true, + ImpactedDependency: "github.com/gin-gonic/gin", + }, fixSupported: true, + }, + { + fixVersionInfo: &utils.FixDetails{ + FixVersion: "1.3.0", + PackageType: coreutils.Go, + DirectDependency: true, + ImpactedDependency: "github.com/google/uuid", + }, fixSupported: true, + }, + } + for _, test := range testcases { + t.Run(test.fixVersionInfo.ImpactedDependency+" direct:"+strconv.FormatBool(test.fixVersionInfo.DirectDependency), func(t *testing.T) { + testDataDir := getTestDataDir(t, test.fixVersionInfo.DirectDependency) + cleanup := createTempDirAndChDir(t, testDataDir, coreutils.Go) + err := goPackageHandler.UpdateDependency(test.fixVersionInfo) + if !test.fixSupported { + assert.Error(t, err, "Expected error to occur") + assert.IsType(t, &utils.ErrUnsupportedFix{}, err, "Expected unsupported fix error") + } else { + assert.NoError(t, err) + assertFixVersionInPackageDescriptor(t, test, "go.mod") + } + assert.NoError(t, os.Chdir(testDataDir)) + cleanup() + }) } +} - testCases := []testCase{ - {fixVersionInfo: utils.FixVersionInfo{FixVersion: "1.2.3", PackageType: "pkg", DirectDependency: true}, newFixVersion: "1.2.4", expectedOutput: "1.2.4"}, - {fixVersionInfo: utils.FixVersionInfo{FixVersion: "1.2.3", PackageType: "pkg", DirectDependency: true}, newFixVersion: "1.0.4", expectedOutput: "1.2.3"}, +// Python, includes pip,pipenv, poetry +func TestPythonPackageHandler_UpdateDependency(t *testing.T) { + testcases := []pythonIndirectDependencies{ + {dependencyFixTest: dependencyFixTest{fixVersionInfo: &utils.FixDetails{ + FixVersion: "1.25.9", PackageType: coreutils.Pip, ImpactedDependency: "urllib3", DirectDependency: false}, fixSupported: false}, requirementsPath: "requirements.txt"}, + {dependencyFixTest: dependencyFixTest{fixVersionInfo: &utils.FixDetails{ + FixVersion: "1.25.9", PackageType: coreutils.Poetry, ImpactedDependency: "urllib3", DirectDependency: false}, fixSupported: false}, requirementsPath: "pyproejct.toml"}, + {dependencyFixTest: dependencyFixTest{fixVersionInfo: &utils.FixDetails{ + FixVersion: "1.25.9", PackageType: coreutils.Pipenv, ImpactedDependency: "urllib3", DirectDependency: false}, fixSupported: false}, requirementsPath: "Pipfile"}, + {dependencyFixTest: dependencyFixTest{fixVersionInfo: &utils.FixDetails{ + FixVersion: "2.4.0", PackageType: coreutils.Pip, ImpactedDependency: "pyjwt", DirectDependency: true}, fixSupported: true}, requirementsPath: "requirements.txt"}, + {dependencyFixTest: dependencyFixTest{fixVersionInfo: &utils.FixDetails{ + FixVersion: "2.4.0", PackageType: coreutils.Pip, ImpactedDependency: "Pyjwt", DirectDependency: true}, fixSupported: true}, requirementsPath: "requirements.txt"}, + {dependencyFixTest: dependencyFixTest{fixVersionInfo: &utils.FixDetails{ + FixVersion: "2.4.0", PackageType: coreutils.Pip, ImpactedDependency: "pyjwt", DirectDependency: true}, fixSupported: true}, requirementsPath: "setup.py"}, + {dependencyFixTest: dependencyFixTest{fixVersionInfo: &utils.FixDetails{ + FixVersion: "2.4.0", PackageType: coreutils.Poetry, ImpactedDependency: "pyjwt", DirectDependency: true}, fixSupported: true}, requirementsPath: "pyproject.toml"}, + {dependencyFixTest: dependencyFixTest{fixVersionInfo: &utils.FixDetails{ + FixVersion: "2.4.0", PackageType: coreutils.Poetry, ImpactedDependency: "pyjwt", DirectDependency: true}, fixSupported: true}, requirementsPath: "pyproject.toml"}, + } + for _, test := range testcases { + t.Run(test.fixVersionInfo.ImpactedDependency+" direct:"+strconv.FormatBool(test.fixVersionInfo.DirectDependency), func(t *testing.T) { + testDataDir := getTestDataDir(t, test.fixVersionInfo.DirectDependency) + pythonPackageHandler := GetCompatiblePackageHandler(test.fixVersionInfo, &utils.ScanDetails{ + Project: &utils.Project{PipRequirementsFile: test.requirementsPath}}) + cleanup := createTempDirAndChDir(t, testDataDir, test.fixVersionInfo.PackageType) + err := pythonPackageHandler.UpdateDependency(test.fixVersionInfo) + if !test.fixSupported { + assert.Error(t, err, "Expected error to occur") + assert.IsType(t, &utils.ErrUnsupportedFix{}, err, "Expected unsupported fix error") + } else { + assert.NoError(t, err) + } + assert.NoError(t, os.Chdir(testDataDir)) + cleanup() + }) } +} - for _, tc := range testCases { - t.Run(tc.expectedOutput, func(t *testing.T) { - tc.fixVersionInfo.UpdateFixVersionIfMax(tc.newFixVersion) - assert.Equal(t, tc.expectedOutput, tc.fixVersionInfo.FixVersion) +func TestPipPackageRegex(t *testing.T) { + var pipPackagesRegexTests = []pipPackageRegexTest{ + {"oslo.config", "oslo.config>=1.12.1,<1.13"}, + {"oslo.utils", "oslo.utils<5.0,>=4.0.0"}, + {"paramiko", "paramiko==2.7.2"}, + {"passlib", "passlib<=1.7.4"}, + {"PassLib", "passlib<=1.7.4"}, + {"prance", "prance>=0.9.0"}, + {"prompt-toolkit", "prompt-toolkit~=1.0.15"}, + {"pyinotify", "pyinotify>0.9.6"}, + {"pyjwt", "pyjwt>1.7.1"}, + {"PyJWT", "pyjwt>1.7.1"}, + {"urllib3", "urllib3 > 1.1.9, < 1.5.*"}, + } + for _, pack := range pipPackagesRegexTests { + re := regexp.MustCompile(PythonPackageRegexPrefix + "(" + pack.packageName + "|" + strings.ToLower(pack.packageName) + ")" + PythonPackageRegexSuffix) + found := re.FindString(requirementsFile) + assert.Equal(t, pack.expectedRequirement, strings.ToLower(found)) + } +} + +// Npm +func TestNpmPackageHandler_UpdateDependency(t *testing.T) { + npmPackageHandler := &NpmPackageHandler{} + testcases := []dependencyFixTest{ + { + fixVersionInfo: &utils.FixDetails{ + FixVersion: "0.8.4", + PackageType: coreutils.Npm, + DirectDependency: false, + ImpactedDependency: "mpath", + }, fixSupported: false, + }, + { + fixVersionInfo: &utils.FixDetails{ + FixVersion: "3.0.2", + PackageType: coreutils.Npm, + DirectDependency: true, + ImpactedDependency: "minimatch", + }, fixSupported: true, + }, + } + for _, test := range testcases { + t.Run(test.fixVersionInfo.ImpactedDependency+" direct:"+strconv.FormatBool(test.fixVersionInfo.DirectDependency), func(t *testing.T) { + testDataDir := getTestDataDir(t, test.fixVersionInfo.DirectDependency) + cleanup := createTempDirAndChDir(t, testDataDir, coreutils.Npm) + err := npmPackageHandler.UpdateDependency(test.fixVersionInfo) + if !test.fixSupported { + assert.Error(t, err, "Expected error to occur") + assert.IsType(t, &utils.ErrUnsupportedFix{}, err, "Expected unsupported fix error") + } else { + assert.NoError(t, err) + } + assert.NoError(t, os.Chdir(testDataDir)) + cleanup() + }) + } +} + +// Yarn +func TestYarnPackageHandler_UpdateDependency(t *testing.T) { + yarnPackageHandler := &YarnPackageHandler{} + testcases := []dependencyFixTest{ + { + fixVersionInfo: &utils.FixDetails{ + FixVersion: "1.2.6", + PackageType: coreutils.Yarn, + DirectDependency: false, + ImpactedDependency: "minimist", + }, fixSupported: false, + }, + { + fixVersionInfo: &utils.FixDetails{ + FixVersion: "1.2.6", + PackageType: coreutils.Yarn, + DirectDependency: true, + ImpactedDependency: "minimist", + }, fixSupported: true, + }, + } + for _, test := range testcases { + t.Run(test.fixVersionInfo.ImpactedDependency+" direct:"+strconv.FormatBool(test.fixVersionInfo.DirectDependency), func(t *testing.T) { + testDataDir := getTestDataDir(t, test.fixVersionInfo.DirectDependency) + cleanup := createTempDirAndChDir(t, testDataDir, coreutils.Yarn) + err := yarnPackageHandler.UpdateDependency(test.fixVersionInfo) + if !test.fixSupported { + assert.Error(t, err, "Expected error to occur") + assert.IsType(t, &utils.ErrUnsupportedFix{}, err, "Expected unsupported fix error") + } else { + assert.NoError(t, err) + } + assert.NoError(t, os.Chdir(testDataDir)) + cleanup() }) } } +// Maven +func TestMavenPackageHandler_UpdateDependency(t *testing.T) { + tests := []dependencyFixTest{ + {fixVersionInfo: &utils.FixDetails{ + FixVersion: "2.7", + PackageType: coreutils.Maven, + ImpactedDependency: "commons-io:commons-io", + DirectDependency: true}, fixSupported: true}, + {fixVersionInfo: &utils.FixDetails{ + FixVersion: "4.3.20", + PackageType: coreutils.Maven, + ImpactedDependency: "org.springframework:spring-core", + DirectDependency: false}, fixSupported: false}, + } + for _, test := range tests { + t.Run(test.fixVersionInfo.ImpactedDependency+" direct:"+strconv.FormatBool(test.fixVersionInfo.DirectDependency), func(t *testing.T) { + mavenPackageHandler := MavenPackageHandler{} + testDataDir := getTestDataDir(t, test.fixVersionInfo.DirectDependency) + cleanup := createTempDirAndChDir(t, testDataDir, coreutils.Maven) + err := mavenPackageHandler.UpdateDependency(test.fixVersionInfo) + if !test.fixSupported { + assert.Error(t, err, "Expected error to occur") + assert.IsType(t, &utils.ErrUnsupportedFix{}, err, "Expected unsupported fix error") + } else { + assert.NoError(t, err) + } + assert.NoError(t, os.Chdir(testDataDir)) + cleanup() + }) + } +} + +// Maven utils functions func TestGetDependenciesFromPomXmlSingleDependency(t *testing.T) { testCases := []string{` org.apache.commons @@ -219,6 +437,27 @@ func TestMavenGavReader(t *testing.T) { assert.Len(t, mvnHandler.pomPaths, 2) } +// General Utils functions +func TestFixVersionInfo_UpdateFixVersionIfMax(t *testing.T) { + type testCase struct { + fixVersionInfo utils.FixDetails + newFixVersion string + expectedOutput string + } + + testCases := []testCase{ + {fixVersionInfo: utils.FixDetails{FixVersion: "1.2.3", PackageType: "pkg", DirectDependency: true}, newFixVersion: "1.2.4", expectedOutput: "1.2.4"}, + {fixVersionInfo: utils.FixDetails{FixVersion: "1.2.3", PackageType: "pkg", DirectDependency: true}, newFixVersion: "1.0.4", expectedOutput: "1.2.3"}, + } + + for _, tc := range testCases { + t.Run(tc.expectedOutput, func(t *testing.T) { + tc.fixVersionInfo.UpdateFixVersionIfMax(tc.newFixVersion) + assert.Equal(t, tc.expectedOutput, tc.fixVersionInfo.FixVersion) + }) + } +} + func TestUpdatePackageVersion(t *testing.T) { testProjectPath := filepath.Join("..", "..", "testdata", "packagehandlers") currDir, err := os.Getwd() @@ -267,3 +506,35 @@ func TestUpdatePropertiesVersion(t *testing.T) { assert.NoError(t, err) assert.Contains(t, string(modifiedPom), "2.39.9") } + +func getTestDataDir(t *testing.T, directDependency bool) string { + var projectDir string + if directDependency { + projectDir = "projects" + } else { + projectDir = "indirect-projects" + } + testdataDir, err := filepath.Abs(filepath.Join("..", "..", "testdata/"+projectDir)) + assert.NoError(t, err) + return testdataDir +} + +func createTempDirAndChDir(t *testing.T, testdataDir string, tech coreutils.Technology) func() { + // Create temp technology project + projectPath := filepath.Join(testdataDir, tech.ToString()) + tmpProjectPath, cleanup := testdatautils.CreateTestProject(t, projectPath) + assert.NoError(t, os.Chdir(tmpProjectPath)) + return cleanup +} + +func assertFixVersionInPackageDescriptor(t *testing.T, test dependencyFixTest, packageDescriptor string) { + file, err := os.ReadFile(packageDescriptor) + assert.NoError(t, err) + if !test.fixSupported { + assert.NotContains(t, string(file), test.fixVersionInfo) + } else { + assert.Contains(t, string(file), test.fixVersionInfo.FixVersion) + // Verify that case-sensitive packages in python are lowered + assert.Contains(t, string(file), strings.ToLower(test.fixVersionInfo.ImpactedDependency)) + } +} diff --git a/commands/utils/packagehandlers/pythonpackagehandler.go b/commands/utils/packagehandlers/pythonpackagehandler.go index 3090847cc..d3599022a 100644 --- a/commands/utils/packagehandlers/pythonpackagehandler.go +++ b/commands/utils/packagehandlers/pythonpackagehandler.go @@ -22,41 +22,53 @@ const ( // PythonPackageHandler Handles all the python package mangers as they share behavior type PythonPackageHandler struct { pipRequirementsFile string - GenericPackageHandler + CommonPackageHandler } -func (py *PythonPackageHandler) UpdateImpactedPackage(impactedPackage string, fixVersionInfo *utils.FixVersionInfo, extraArgs ...string) error { - switch fixVersionInfo.PackageType { +func (py *PythonPackageHandler) UpdateDependency(fixDetails *utils.FixDetails) error { + if fixDetails.DirectDependency { + return py.updateDirectDependency(fixDetails) + } else { + return &utils.ErrUnsupportedFix{ + PackageName: fixDetails.ImpactedDependency, + FixedVersion: fixDetails.FixVersion, + ErrorType: utils.IndirectDependencyFixNotSupported, + } + } +} + +func (py *PythonPackageHandler) updateDirectDependency(fixDetails *utils.FixDetails, extraArgs ...string) (err error) { + switch fixDetails.PackageType { case coreutils.Poetry: - return py.handlePoetry(impactedPackage, fixVersionInfo) + return py.handlePoetry(fixDetails) case coreutils.Pip: - return py.handlePip(impactedPackage, fixVersionInfo) + return py.handlePip(fixDetails) case coreutils.Pipenv: - return py.GenericPackageHandler.UpdateImpactedPackage(impactedPackage, fixVersionInfo, extraArgs...) + return py.CommonPackageHandler.UpdateDependency(fixDetails, extraArgs...) default: - return errors.New("unknown python package manger: " + fixVersionInfo.PackageType.GetPackageType()) + return errors.New("unknown python package manger: " + fixDetails.PackageType.GetPackageType()) } } -func (py *PythonPackageHandler) handlePoetry(impactedPackage string, fixVersionInfo *utils.FixVersionInfo) error { +func (py *PythonPackageHandler) handlePoetry(fixDetails *utils.FixDetails) (err error) { // Install the desired fixed version - if err := py.GenericPackageHandler.UpdateImpactedPackage(impactedPackage, fixVersionInfo); err != nil { - return err + if err = py.CommonPackageHandler.UpdateDependency(fixDetails); err != nil { + return } // Update Poetry lock file as well return runPackageMangerCommand(coreutils.Poetry.GetExecCommandName(), []string{"update"}) } -func (py *PythonPackageHandler) handlePip(impactedPackage string, info *utils.FixVersionInfo) error { +func (py *PythonPackageHandler) handlePip(fixDetails *utils.FixDetails) (err error) { var fixedFile string // This function assumes that the version of the dependencies is statically pinned in the requirements file or inside the 'install_requires' array in the setup.py file - fixedPackage := impactedPackage + "==" + info.FixVersion + fixedPackage := fixDetails.ImpactedDependency + "==" + fixDetails.FixVersion if py.pipRequirementsFile == "" { py.pipRequirementsFile = "setup.py" } wd, err := os.Getwd() if err != nil { - return err + return } fullPath := filepath.Join(wd, py.pipRequirementsFile) if !strings.HasPrefix(filepath.Clean(fullPath), wd) { @@ -64,18 +76,18 @@ func (py *PythonPackageHandler) handlePip(impactedPackage string, info *utils.Fi } data, err := os.ReadFile(filepath.Clean(py.pipRequirementsFile)) if err != nil { - return err + return } currentFile := string(data) // Check both original and lowered package name and replace to only one lowered result // This regex will match the impactedPackage with it's pinned version e.py. PyJWT==1.7.1 - re := regexp.MustCompile(PythonPackageRegexPrefix + "(" + impactedPackage + "|" + strings.ToLower(impactedPackage) + ")" + PythonPackageRegexSuffix) + re := regexp.MustCompile(PythonPackageRegexPrefix + "(" + fixDetails.ImpactedDependency + "|" + strings.ToLower(fixDetails.ImpactedDependency) + ")" + PythonPackageRegexSuffix) if packageToReplace := re.FindString(currentFile); packageToReplace != "" { fixedFile = strings.Replace(currentFile, packageToReplace, strings.ToLower(fixedPackage), 1) } if fixedFile == "" { - return fmt.Errorf("impacted package %s not found, fix failed", impactedPackage) + return fmt.Errorf("impacted package %s not found, fix failed", fixDetails.ImpactedDependency) } return os.WriteFile(py.pipRequirementsFile, []byte(fixedFile), 0600) } diff --git a/commands/utils/packagehandlers/yarnpackagehandler.go b/commands/utils/packagehandlers/yarnpackagehandler.go new file mode 100644 index 000000000..406bcb217 --- /dev/null +++ b/commands/utils/packagehandlers/yarnpackagehandler.go @@ -0,0 +1,23 @@ +package packagehandlers + +import "github.com/jfrog/frogbot/commands/utils" + +type YarnPackageHandler struct { + CommonPackageHandler +} + +func (yarn *YarnPackageHandler) UpdateDependency(fixDetails *utils.FixDetails) error { + if fixDetails.DirectDependency { + return yarn.updateDirectDependency(fixDetails) + } else { + return &utils.ErrUnsupportedFix{ + PackageName: fixDetails.ImpactedDependency, + FixedVersion: fixDetails.FixVersion, + ErrorType: utils.IndirectDependencyFixNotSupported, + } + } +} + +func (yarn *YarnPackageHandler) updateDirectDependency(fixDetails *utils.FixDetails, extraArgs ...string) (err error) { + return yarn.CommonPackageHandler.UpdateDependency(fixDetails, extraArgs...) +} diff --git a/commands/utils/params.go b/commands/utils/params.go index 7b7e41819..695a04159 100644 --- a/commands/utils/params.go +++ b/commands/utils/params.go @@ -24,10 +24,12 @@ const ( FrogbotConfigFile = "frogbot-config.yml" ) -var errFrogbotConfigNotFound = fmt.Errorf("%s wasn't found in the Frogbot directory and its subdirectories. Assuming all the configuration is stored as environment variables", FrogbotConfigFile) +var ( + errFrogbotConfigNotFound = fmt.Errorf("%s wasn't found in the Frogbot directory and its subdirectories. Assuming all the configuration is stored as environment variables", FrogbotConfigFile) -// Possible Config file path's to Frogbot Management repository -var osFrogbotConfigPath = filepath.Join(frogbotConfigDir, FrogbotConfigFile) + // Possible Config file path's to Frogbot Management repository + osFrogbotConfigPath = filepath.Join(frogbotConfigDir, FrogbotConfigFile) +) type FrogbotUtils struct { ConfigAggregator FrogbotConfigAggregator diff --git a/commands/utils/utils.go b/commands/utils/utils.go index 954009352..fd0718725 100644 --- a/commands/utils/utils.go +++ b/commands/utils/utils.go @@ -34,7 +34,9 @@ const ( branchCharsMaxLength = 255 branchInvalidLength = "branch name length exceeded " + string(rune(branchCharsMaxLength)) + " chars" invalidBranchTemplate = "branch template must contain " + BranchHashPlaceHolder + " placeholder " - SkipIndirectVulnerabilitiesMsg = "%s is an indirect dependency that will not be updated to version %s.\nFixing indirect dependencies can introduce conflicts with other dependencies that rely on the previous version.\nFrogbot skips this to avoid potential incompatibilities." + skipIndirectVulnerabilitiesMsg = "%s is an indirect dependency that will not be updated to version %s.\nFixing indirect dependencies can introduce conflicts with other dependencies that rely on the previous version.\nFrogbot skips this to avoid potential incompatibilities." + skipBuildToolDependencyMsg = "Skipping vulnerable package %s since it is not defined in your package descriptor file. " + + "Update %s version to %s to fix this vulnerability." ) var ( @@ -48,30 +50,45 @@ var BuildToolsDependenciesMap = map[coreutils.Technology][]string{ coreutils.Pip: {"pip", "setuptools", "wheel"}, } -type ErrUnsupportedIndirectFix struct { +type ErrUnsupportedFix struct { PackageName string FixedVersion string + ErrorType UnsupportedErrorType } -func (e *ErrUnsupportedIndirectFix) Error() string { - return fmt.Sprintf(SkipIndirectVulnerabilitiesMsg, e.PackageName, e.FixedVersion) +// Custom error for unsupported fixes +// Currently we hold two unsupported reasons, indirect and build tools dependencies. +func (err *ErrUnsupportedFix) Error() string { + switch err.ErrorType { + case IndirectDependencyFixNotSupported: + return fmt.Sprintf(skipIndirectVulnerabilitiesMsg, err.PackageName, err.FixedVersion) + case BuildToolsDependencyFixNotSupported: + return fmt.Sprintf(skipBuildToolDependencyMsg, err.PackageName, err.PackageName, err.FixedVersion) + default: + panic("Incompatible custom error!") + } } -// FixVersionInfo is a basic struct used to hold needed information about version fixing -type FixVersionInfo struct { - FixVersion string - PackageType coreutils.Technology +// FixDetails is a basic struct used to hold needed information for fixing vulnerabilities +type FixDetails struct { + // Package technology + PackageType coreutils.Technology + // Name of the impacted dependency + ImpactedDependency string + // Suggested fix version + FixVersion string + // Is the dependency direct or transitive? DirectDependency bool } -func NewFixVersionInfo(newFixVersion string, packageType coreutils.Technology, directDependency bool) *FixVersionInfo { - return &FixVersionInfo{newFixVersion, packageType, directDependency} +func NewFixDetails(fixVersion string, packageType coreutils.Technology, directDependency bool, impactedPackage string) *FixDetails { + return &FixDetails{packageType, impactedPackage, fixVersion, directDependency} } -func (fvi *FixVersionInfo) UpdateFixVersionIfMax(newFixVersion string) { +func (fvi *FixDetails) UpdateFixVersionIfMax(fixVersion string) { // Update fvi.FixVersion as the maximum version if found a new version that is greater than the previous maximum version. - if fvi.FixVersion == "" || version.NewVersion(fvi.FixVersion).Compare(newFixVersion) > 0 { - fvi.FixVersion = newFixVersion + if fvi.FixVersion == "" || version.NewVersion(fvi.FixVersion).Compare(fixVersion) > 0 { + fvi.FixVersion = fixVersion } } @@ -167,7 +184,7 @@ func CommitStatusToString(status vcsclient.CommitStatus) string { // Generates MD5Hash from a FixVersionMap object // The map can be returned in different order from Xray, so we need to sort the strings before hashing. -func fixVersionsMapToMd5Hash(versionsMap map[string]*FixVersionInfo) (string, error) { +func fixVersionsMapToMd5Hash(versionsMap map[string]*FixDetails) (string, error) { h := crypto.MD5.New() // Sort the package names keys := make([]string, 0, len(versionsMap)) diff --git a/commands/utils/utils_test.go b/commands/utils/utils_test.go index 1e207d355..f4970f7f3 100644 --- a/commands/utils/utils_test.go +++ b/commands/utils/utils_test.go @@ -131,27 +131,27 @@ func TestMd5Hash(t *testing.T) { func TestFixVersionsMapToMd5Hash(t *testing.T) { tests := []struct { - fixVersionMap map[string]*FixVersionInfo + fixVersionMap map[string]*FixDetails expectedHash string }{ { - fixVersionMap: map[string]*FixVersionInfo{ + fixVersionMap: map[string]*FixDetails{ "pkg": {FixVersion: "1.2.3", PackageType: coreutils.Npm, DirectDependency: false}}, expectedHash: "0aa066970b613b114f8e21d11c74ff94", }, { - fixVersionMap: map[string]*FixVersionInfo{ + fixVersionMap: map[string]*FixDetails{ "pkg": {FixVersion: "5.2.3", PackageType: coreutils.Go, DirectDependency: false}, "pkg2": {FixVersion: "1.2.3", PackageType: coreutils.Go, DirectDependency: false}}, expectedHash: "a0d4119dfe5fc5186d6c2cf1497f8c7c", }, { // The Same map with different order should be the same hash. - fixVersionMap: map[string]*FixVersionInfo{ + fixVersionMap: map[string]*FixDetails{ "pkg2": {FixVersion: "1.2.3", PackageType: coreutils.Go, DirectDependency: false}, "pkg": {FixVersion: "5.2.3", PackageType: coreutils.Go, DirectDependency: false}}, expectedHash: "a0d4119dfe5fc5186d6c2cf1497f8c7c", }, { - fixVersionMap: map[string]*FixVersionInfo{ + fixVersionMap: map[string]*FixDetails{ "myNuget": {FixVersion: "0.2.33", PackageType: coreutils.Nuget, DirectDependency: false}}, expectedHash: "887ac2c931920c20956409702c0dfbc7", }, From 7f3f31eca868c02d0109877c05dd0c4cc3550fd6 Mon Sep 17 00:00:00 2001 From: Eyal Delarea Date: Mon, 22 May 2023 18:02:56 +0300 Subject: [PATCH 2/6] Update ReadMe & Github Workflow (#344) --- .github/workflows/frogbot-scan-and-fix.yml | 91 +++++++++++++++++-- README.md | 7 +- .../scan-and-fix/frogbot-scan-and-fix-go.yml | 2 +- .../frogbot-scan-pr-dotnet.yml | 2 +- .../scan-pull-request/frogbot-scan-pr-go.yml | 4 +- .../frogbot-scan-pr-gradle.yml | 2 +- .../frogbot-scan-pr-maven.yml | 2 +- .../scan-pull-request/frogbot-scan-pr-npm.yml | 2 +- .../frogbot-scan-pr-nuget.yml | 2 +- .../scan-pull-request/frogbot-scan-pr-pip.yml | 2 +- .../frogbot-scan-pr-pipenv.yml | 2 +- .../frogbot-scan-pr-poetry.yml | 2 +- .../frogbot-scan-pr-yarn.yml | 2 +- .../code-scanning/frogbot-scan-pr.yml | 2 +- 14 files changed, 104 insertions(+), 20 deletions(-) diff --git a/.github/workflows/frogbot-scan-and-fix.yml b/.github/workflows/frogbot-scan-and-fix.yml index b49b919c2..510651bb3 100644 --- a/.github/workflows/frogbot-scan-and-fix.yml +++ b/.github/workflows/frogbot-scan-and-fix.yml @@ -1,11 +1,7 @@ name: "Frogbot Scan and Fix" on: - push: - branches: - - "dev" - tags-ignore: - - '**' schedule: + # The repository will be scanned once a day at 00:00 GMT. - cron: "0 0 * * *" permissions: contents: write @@ -14,8 +10,14 @@ permissions: jobs: create-fix-pull-requests: runs-on: ubuntu-latest + strategy: + matrix: + # The repository scanning will be triggered periodically on the following branches. + branch: [ "dev" ] steps: - uses: actions/checkout@v3 + with: + ref: ${{ matrix.branch }} # Install prerequisites - name: Setup Go @@ -26,13 +28,90 @@ jobs: - uses: jfrog/frogbot@v2 env: # [Mandatory] - # JFrog platform URL (This functionality requires version 3.29.0 or above of Xray) + # JFrog platform URL JF_URL: ${{ secrets.FROGBOT_URL }} # [Mandatory if JF_USER and JF_PASSWORD are not provided] # JFrog access token with 'read' permissions on Xray service JF_ACCESS_TOKEN: ${{ secrets.FROGBOT_ACCESS_TOKEN }} + # [Mandatory if JF_ACCESS_TOKEN is not provided] + # JFrog username with 'read' permissions for Xray. Must be provided with JF_PASSWORD + # JF_USER: ${{ secrets.JF_USER }} + + # [Mandatory if JF_ACCESS_TOKEN is not provided] + # JFrog password. Must be provided with JF_USER + # JF_PASSWORD: ${{ secrets.JF_PASSWORD }} + # [Mandatory] # The GitHub token automatically generated for the job JF_GIT_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + # [Optional, default: https://api.github.com] + # API endpoint to GitHub + # JF_GIT_API_ENDPOINT: https://github.example.com + + # [Optional] + # If the machine that runs Frogbot has no access to the internet, set the name of a remote repository + # in Artifactory, which proxies https://releases.jfrog.io + # The 'frogbot' executable and other tools it needs will be downloaded through this repository. + # JF_RELEASES_REPO: "" + + + + ########################################################################## + ## If your project uses a 'frogbot-config.yml' file, you can define ## + ## the following variables inside the file, instead of here. ## + ########################################################################## + + # [Optional, default: "."] + # Relative path to the root of the project in the Git repository + # JF_WORKING_DIR: path/to/project/dir + + # [Optional] + # Xray Watches. Learn more about them here: https://www.jfrog.com/confluence/display/JFROG/Configuring+Xray+Watches + # JF_WATCHES: ,... + + # [Optional] + # JFrog project. Learn more about it here: https://www.jfrog.com/confluence/display/JFROG/Projects + # JF_PROJECT: + + # [Optional, default: "TRUE"] + # Fails the Frogbot task if any security issue is found. + # JF_FAIL: "FALSE" + + # [Optional] + # Frogbot will download the project dependencies, if they're not cached locally. To download the + # dependencies from a virtual repository in Artifactory, set the name of the repository. There's no + # need to set this value, if it is set in the frogbot-config.yml file. + # JF_DEPS_REPO: "" + + # [Optional] + # Template for the branch name generated by Frogbot when creating pull requests with fixes. + # The template must include ${BRANCH_NAME_HASH}, to ensure that the generated branch name is unique. + # The template can optionally include the ${IMPACTED_PACKAGE} and ${FIX_VERSION} variables. + # JF_BRANCH_NAME_TEMPLATE: "frogbot-${IMPACTED_PACKAGE}-${BRANCH_NAME_HASH}" + + # [Optional] + # Template for the commit message generated by Frogbot when creating pull requests with fixes + # The template can optionally include the ${IMPACTED_PACKAGE} and ${FIX_VERSION} variables. + # JF_COMMIT_MESSAGE_TEMPLATE: "Upgrade ${IMPACTED_PACKAGE} to ${FIX_VERSION}" + + # [Optional] + # Template for the pull request title generated by Frogbot when creating pull requests with fixes. + # The template can optionally include the ${IMPACTED_PACKAGE} and ${FIX_VERSION} variables. + # JF_PULL_REQUEST_TITLE_TEMPLATE: "[🐸 Frogbot] Upgrade ${IMPACTED_PACKAGE} to to ${FIX_VERSION}" + + # [Optional, Default: "FALSE"] + # If TRUE, Frogbot creates a single pull request with all the fixes. + # If FALSE, Frogbot creates a separate pull request for each fix. + # JF_GIT_AGGREGATE_FIXES: "FALSE" + + # [Optional, Default: "FALSE"] + # Handle vulnerabilities with fix versions only + # JF_FIXABLE_ONLY: "TRUE" + + # [Optional] + # Set the minimum severity for vulnerabilities that should be fixed and commented on in pull requests + # The following values are accepted: Low, Medium, High or Critical + # JF_MIN_SEVERITY: "" diff --git a/README.md b/README.md index 43653d889..281780e19 100644 --- a/README.md +++ b/README.md @@ -4,9 +4,14 @@ [![Frogbot](images/frogbot-intro.png)](#readme) -[![Scanned by Frogbot](https://raw.github.com/jfrog/frogbot/master/images/frogbot-badge.svg)](https://github.com/jfrog/frogbot#readme) [![Build status](https://github.com/jfrog/frogbot/actions/workflows/test.yml/badge.svg)](https://github.com/jfrog/frogbot/actions/workflows/test.yml) [![GitHub Action Test](https://github.com/jfrog/frogbot/actions/workflows/action-test.yml/badge.svg)](https://github.com/jfrog/frogbot/actions/workflows/action-test.yml) +[![Scanned by Frogbot](https://raw.github.com/jfrog/frogbot/master/images/frogbot-badge.svg)](https://github.com/jfrog/frogbot#readme) [![Go Report Card](https://goreportcard.com/badge/github.com/jfrog/frogbot)](https://goreportcard.com/report/github.com/jfrog/frogbot) +| Branch | Status | +|:------:|:----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------:| +| master | [![Build status](https://github.com/jfrog/frogbot/actions/workflows/test.yml/badge.svg?branch=master)](https://github.com/jfrog/frogbot/actions/workflows/test.yml?branch=master) [![GitHub Action Test](https://github.com/jfrog/frogbot/actions/workflows/action-test.yml/badge.svg?branch=master)](https://github.com/jfrog/frogbot/actions/workflows/action-test.yml?branch=master) | +| dev | [![Build status](https://github.com/jfrog/frogbot/actions/workflows/test.yml/badge.svg?branch=dev)](https://github.com/jfrog/frogbot/actions/workflows/test.yml?branch=dev) [![GitHub Action Test](https://github.com/jfrog/frogbot/actions/workflows/action-test.yml/badge.svg?branch=dev)](https://github.com/jfrog/frogbot/actions/workflows/action-test.yml?branch=dev) | + ## Table of contents diff --git a/docs/templates/github-actions/scan-and-fix/frogbot-scan-and-fix-go.yml b/docs/templates/github-actions/scan-and-fix/frogbot-scan-and-fix-go.yml index 9f6aea130..3b229b72e 100644 --- a/docs/templates/github-actions/scan-and-fix/frogbot-scan-and-fix-go.yml +++ b/docs/templates/github-actions/scan-and-fix/frogbot-scan-and-fix-go.yml @@ -21,7 +21,7 @@ jobs: # Install prerequisites - name: Setup Go - uses: actions/setup-go@v2 + uses: actions/setup-go@v3 with: go-version: 1.20.x diff --git a/docs/templates/github-actions/scan-pull-request/frogbot-scan-pr-dotnet.yml b/docs/templates/github-actions/scan-pull-request/frogbot-scan-pr-dotnet.yml index 2f41bc761..4df3d608f 100644 --- a/docs/templates/github-actions/scan-pull-request/frogbot-scan-pr-dotnet.yml +++ b/docs/templates/github-actions/scan-pull-request/frogbot-scan-pr-dotnet.yml @@ -12,7 +12,7 @@ jobs: # "frogbot" GitHub environment can approve the pull request to be scanned. environment: frogbot steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 with: ref: ${{ github.event.pull_request.head.sha }} diff --git a/docs/templates/github-actions/scan-pull-request/frogbot-scan-pr-go.yml b/docs/templates/github-actions/scan-pull-request/frogbot-scan-pr-go.yml index d1fd6d44a..3a69d0630 100644 --- a/docs/templates/github-actions/scan-pull-request/frogbot-scan-pr-go.yml +++ b/docs/templates/github-actions/scan-pull-request/frogbot-scan-pr-go.yml @@ -12,13 +12,13 @@ jobs: # "frogbot" GitHub environment can approve the pull request to be scanned. environment: frogbot steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 with: ref: ${{ github.event.pull_request.head.sha }} # Install prerequisites - name: Setup Go - uses: actions/setup-go@v2 + uses: actions/setup-go@v3 with: go-version: 1.20.x diff --git a/docs/templates/github-actions/scan-pull-request/frogbot-scan-pr-gradle.yml b/docs/templates/github-actions/scan-pull-request/frogbot-scan-pr-gradle.yml index db87f7ee0..d8e152786 100644 --- a/docs/templates/github-actions/scan-pull-request/frogbot-scan-pr-gradle.yml +++ b/docs/templates/github-actions/scan-pull-request/frogbot-scan-pr-gradle.yml @@ -12,7 +12,7 @@ jobs: # "frogbot" GitHub environment can approve the pull request to be scanned. environment: frogbot steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 with: ref: ${{ github.event.pull_request.head.sha }} diff --git a/docs/templates/github-actions/scan-pull-request/frogbot-scan-pr-maven.yml b/docs/templates/github-actions/scan-pull-request/frogbot-scan-pr-maven.yml index 099b47bd2..701e0a559 100644 --- a/docs/templates/github-actions/scan-pull-request/frogbot-scan-pr-maven.yml +++ b/docs/templates/github-actions/scan-pull-request/frogbot-scan-pr-maven.yml @@ -12,7 +12,7 @@ jobs: # "frogbot" GitHub environment can approve the pull request to be scanned. environment: frogbot steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 with: ref: ${{ github.event.pull_request.head.sha }} diff --git a/docs/templates/github-actions/scan-pull-request/frogbot-scan-pr-npm.yml b/docs/templates/github-actions/scan-pull-request/frogbot-scan-pr-npm.yml index 4d702d853..61646535d 100644 --- a/docs/templates/github-actions/scan-pull-request/frogbot-scan-pr-npm.yml +++ b/docs/templates/github-actions/scan-pull-request/frogbot-scan-pr-npm.yml @@ -12,7 +12,7 @@ jobs: # "frogbot" GitHub environment can approve the pull request to be scanned. environment: frogbot steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 with: ref: ${{ github.event.pull_request.head.sha }} diff --git a/docs/templates/github-actions/scan-pull-request/frogbot-scan-pr-nuget.yml b/docs/templates/github-actions/scan-pull-request/frogbot-scan-pr-nuget.yml index 0c2ab0b6f..f39cff82f 100644 --- a/docs/templates/github-actions/scan-pull-request/frogbot-scan-pr-nuget.yml +++ b/docs/templates/github-actions/scan-pull-request/frogbot-scan-pr-nuget.yml @@ -12,7 +12,7 @@ jobs: # "frogbot" GitHub environment can approve the pull request to be scanned. environment: frogbot steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 with: ref: ${{ github.event.pull_request.head.sha }} diff --git a/docs/templates/github-actions/scan-pull-request/frogbot-scan-pr-pip.yml b/docs/templates/github-actions/scan-pull-request/frogbot-scan-pr-pip.yml index 8e178ac58..279fcbec3 100644 --- a/docs/templates/github-actions/scan-pull-request/frogbot-scan-pr-pip.yml +++ b/docs/templates/github-actions/scan-pull-request/frogbot-scan-pr-pip.yml @@ -12,7 +12,7 @@ jobs: # "frogbot" GitHub environment can approve the pull request to be scanned. environment: frogbot steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 with: ref: ${{ github.event.pull_request.head.sha }} diff --git a/docs/templates/github-actions/scan-pull-request/frogbot-scan-pr-pipenv.yml b/docs/templates/github-actions/scan-pull-request/frogbot-scan-pr-pipenv.yml index 34eadf9a0..8b85356c3 100644 --- a/docs/templates/github-actions/scan-pull-request/frogbot-scan-pr-pipenv.yml +++ b/docs/templates/github-actions/scan-pull-request/frogbot-scan-pr-pipenv.yml @@ -12,7 +12,7 @@ jobs: # "frogbot" GitHub environment can approve the pull request to be scanned. environment: frogbot steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 with: ref: ${{ github.event.pull_request.head.sha }} diff --git a/docs/templates/github-actions/scan-pull-request/frogbot-scan-pr-poetry.yml b/docs/templates/github-actions/scan-pull-request/frogbot-scan-pr-poetry.yml index b0235b017..4b20a2bae 100644 --- a/docs/templates/github-actions/scan-pull-request/frogbot-scan-pr-poetry.yml +++ b/docs/templates/github-actions/scan-pull-request/frogbot-scan-pr-poetry.yml @@ -12,7 +12,7 @@ jobs: # "frogbot" GitHub environment can approve the pull request to be scanned. environment: frogbot steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 with: ref: ${{ github.event.pull_request.head.sha }} diff --git a/docs/templates/github-actions/scan-pull-request/frogbot-scan-pr-yarn.yml b/docs/templates/github-actions/scan-pull-request/frogbot-scan-pr-yarn.yml index acd07b9fc..962adc490 100644 --- a/docs/templates/github-actions/scan-pull-request/frogbot-scan-pr-yarn.yml +++ b/docs/templates/github-actions/scan-pull-request/frogbot-scan-pr-yarn.yml @@ -12,7 +12,7 @@ jobs: # "frogbot" GitHub environment can approve the pull request to be scanned. environment: frogbot steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 with: ref: ${{ github.event.pull_request.head.sha }} diff --git a/starter-workflows/code-scanning/frogbot-scan-pr.yml b/starter-workflows/code-scanning/frogbot-scan-pr.yml index ab0b057fb..2dc0a2ac3 100644 --- a/starter-workflows/code-scanning/frogbot-scan-pr.yml +++ b/starter-workflows/code-scanning/frogbot-scan-pr.yml @@ -24,7 +24,7 @@ jobs: # Read more here (Install Frogbot Using GitHub Actions): https://github.com/jfrog/frogbot/blob/master/docs/install-github.md environment: frogbot steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 with: ref: ${{ github.event.pull_request.head.sha }} From 98b90c5020ba9ccc4ac5d80e02f3f5c6978e8bc3 Mon Sep 17 00:00:00 2001 From: Tal Arian Date: Wed, 24 May 2023 19:01:40 +0300 Subject: [PATCH 3/6] Adjust extractors downloading (#346) --- commands/utils/extractors.go | 2 +- go.mod | 7 ++++--- go.sum | 9 +++++---- 3 files changed, 10 insertions(+), 8 deletions(-) diff --git a/commands/utils/extractors.go b/commands/utils/extractors.go index eea93d3ab..656fe4638 100644 --- a/commands/utils/extractors.go +++ b/commands/utils/extractors.go @@ -64,7 +64,7 @@ func downloadExtractor(remoteRepoName string, server *config.ServerDetails, extr log.Info("Downloading", extractor.extractorType, "extractor to path:", extractor.localPath) remoteServer := getRemoteServer(server, remoteRepoName) - return utils.DownloadExtractor(remoteServer, extractor.downloadFromPath(), extractor.downloadToPath()) + return utils.DownloadDependency(remoteServer, extractor.downloadFromPath(), extractor.downloadToPath(), false) } func getRemoteServer(server *config.ServerDetails, remoteName string) *config.ServerDetails { diff --git a/go.mod b/go.mod index 108738753..eb19bb6f7 100644 --- a/go.mod +++ b/go.mod @@ -8,8 +8,8 @@ require ( github.com/jfrog/build-info-go v1.9.6 github.com/jfrog/froggit-go v1.7.3 github.com/jfrog/gofrog v1.3.0 - github.com/jfrog/jfrog-cli-core/v2 v2.34.0 - github.com/jfrog/jfrog-client-go v1.28.6 + github.com/jfrog/jfrog-cli-core/v2 v2.34.4 + github.com/jfrog/jfrog-client-go v1.29.1 github.com/mholt/archiver/v3 v3.5.1 github.com/pkg/errors v0.9.1 github.com/stretchr/testify v1.8.2 @@ -101,6 +101,7 @@ require ( golang.org/x/crypto v0.9.0 // indirect golang.org/x/net v0.10.0 // indirect golang.org/x/oauth2 v0.0.0-20221014153046-6fdb5e3db783 // indirect + golang.org/x/sync v0.1.0 // indirect golang.org/x/sys v0.8.0 // indirect golang.org/x/term v0.8.0 // indirect golang.org/x/text v0.9.0 // indirect @@ -111,7 +112,7 @@ require ( gopkg.in/warnings.v0 v0.1.2 // indirect ) -//replace github.com/jfrog/jfrog-cli-core/v2 => github.com/jfrog/jfrog-cli-core/v2 v2.31.1-0.20230521154339-4d1c76bd8a25 +replace github.com/jfrog/jfrog-cli-core/v2 => github.com/jfrog/jfrog-cli-core/v2 v2.31.1-0.20230524155422-b3ac4175049a //replace github.com/jfrog/build-info-go => github.com/jfrog/build-info-go v1.8.9-0.20230518114837-fe6a826d5001 diff --git a/go.sum b/go.sum index db4b4d206..d13c23471 100644 --- a/go.sum +++ b/go.sum @@ -230,10 +230,10 @@ github.com/jfrog/froggit-go v1.7.3 h1:ZOXbFsdmtM6+uDbo1U+elV7sPZ4EN6grDck9aHqKyn github.com/jfrog/froggit-go v1.7.3/go.mod h1:xfsfQXzSaAM04RV9IyU5heBiRrsm2oS6rFCfEofQr6U= github.com/jfrog/gofrog v1.3.0 h1:o4zgsBZE4QyDbz2M7D4K6fXPTBJht+8lE87mS9bw7Gk= github.com/jfrog/gofrog v1.3.0/go.mod h1:IFMc+V/yf7rA5WZ74CSbXe+Lgf0iApEQLxRZVzKRUR0= -github.com/jfrog/jfrog-cli-core/v2 v2.34.0 h1:D+79G7DSnnQrr6dG4E3HRO44mKiTdOogRt5F0AhWoSU= -github.com/jfrog/jfrog-cli-core/v2 v2.34.0/go.mod h1:h0K7n9gh5RNN6/1IVIufCV1OgP3JouN03Nqw2XlDkT4= -github.com/jfrog/jfrog-client-go v1.28.6 h1:7r221cxmBOAWuiazY5X2BTxNXCqh2IFd+retDh7xX2Y= -github.com/jfrog/jfrog-client-go v1.28.6/go.mod h1:slAmJkUwpPpy6o+rEb5pg2DGdcpME2yz5SuSz6NHJ7o= +github.com/jfrog/jfrog-cli-core/v2 v2.31.1-0.20230524155422-b3ac4175049a h1:314Ti3MclykCssxZvJjbLnBZgaNknOw8hEaWmMLAt+0= +github.com/jfrog/jfrog-cli-core/v2 v2.31.1-0.20230524155422-b3ac4175049a/go.mod h1:htVCrvgB2P3ROJNQEPs2LptVTQ33l8349q9t/zD/hAM= +github.com/jfrog/jfrog-client-go v1.29.1 h1:R5NyZ6qbroY8uG6vWX/5nLjATMo8OMOhyVd3GcejFDI= +github.com/jfrog/jfrog-client-go v1.29.1/go.mod h1:nGUoz5Qi9kTP0VfkvOVJ3nudsD3dq3y/d0sLjlkpnrI= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= github.com/k0kubun/colorstring v0.0.0-20150214042306-9440f1994b88/go.mod h1:3w7q1U84EfirKl04SVQ/s7nPm1ZPhiXd34z40TNz36k= @@ -517,6 +517,7 @@ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180224232135-f6cff0780e54/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= From 616dfa01965bc31459029e7087c08a5e291e7938 Mon Sep 17 00:00:00 2001 From: Eyal Ben Moshe Date: Thu, 25 May 2023 16:01:17 +0300 Subject: [PATCH 4/6] Documentation updates (#347) --- README.md | 172 ++++++++-------- docs/install-bitbucket-server.md | 327 ++++++++++++++++--------------- 2 files changed, 253 insertions(+), 246 deletions(-) diff --git a/README.md b/README.md index 281780e19..acd4c7e5e 100644 --- a/README.md +++ b/README.md @@ -16,28 +16,41 @@ ## Table of contents -- [Frogbot](#frogbot) - - [Table of contents](#table-of-contents) - - [🤖 What is Frogbot?](#-what-is-frogbot) - - [🖥️ Installing Frogbot](#️-installing-frogbot) - - [🚥 Using Frogbot](#-using-frogbot) - - [Scanning pull requests when they are opened](#scanning-pull-requests-when-they-are-opened) - - [General](#general) - - [🕵️‍♀️ How does Pull Request scanning work?](#️️-how-does-pull-request-scanning-work) - - [👮 Security note for pull requests scanning](#-security-note-for-pull-requests-scanning) - - [Scan results](#scan-results) - - [👍 No issues](#-no-issues) - - [👎 Issues were found](#-issues-were-found) - - [Scanning repositories and fixing issues](#scanning-repositories-and-fixing-issues) - - [📛 Adding the Frogbot badge](#-adding-the-frogbot-badge) - - [🔥 Reporting issues](#-reporting-issues) - - [💻 Contributions](#-contributions) +- [🤖 About JFrog Frogbot](#-about-jfrog-frogbot) +- [🖥️ Installing Frogbot](#️-installing-frogbot) +- [🚥 Using Frogbot](#-using-frogbot) + - [Scanning pull requests](#scanning-pull-requests) + - [Scanning repositories and fixing issues](#scanning-repositories-and-fixing-issues) +- [📛 Adding the Frogbot badge](#-adding-the-frogbot-badge) +- [🔥 Reporting issues](#-reporting-issues) +- [💻 Contributions](#-contributions)
-## 🤖 What is Frogbot? +## 🤖 About JFrog Frogbot +### Overview -Frogbot is a Git bot that scans your pull requests and repositories for security vulnerabilities. You can scan pull requests when they are opened, and Git repositories following new commits. +JFrog Frogbot is a Git bot that scans your git repositories for security vulnerabilities. +- Frogbot scans pull requests right after they are opened, but before they are merged. This unique capability ensures that the code is scanned and can be fixed even before vulnerabilities are introduced in the code base. +- Frogbot scans the Git repository periodically and creates pull requests with fixes for vulnerabilities that are detected. + +Frogbot uses JFrog's vast vulnerabilities database, to which we continuously add new component vulnerability data. Also included is VulnDB, the industry's most comprehensive security database, to further extend the range of vulnerabilities detected and fixed by Frogbot. + +Frogbot supports the following Git providers: +- Azure Repos +- Bitbucket Server +- GitHub +- GitLab. + +### What's needed for the set up? + +- Frogbot uses a JFrog environment to scan your Git repositories. If you don't have a JFrog environment, you can set up one for free, and use it with no limits. +- Frogbot also requires a runtime environment for the scanning. The following environments are supported: + + - GitHub Actions + - JFrog Pipelines + - Jenkins + - Azure Pipelines ## 🖥️ Installing Frogbot @@ -87,18 +100,11 @@ After the setup is complete, you'll receive an email with your JFrog environment
## 🚥 Using Frogbot -### Scanning pull requests when they are opened +### Scanning pull requests #### General Frogbot uses [JFrog Xray](https://jfrog.com/xray/) (version 3.29.0 and above is required) to scan your pull requests. It adds the scan results as a comment on the pull request. If no new vulnerabilities are found, Frogbot will also add a comment, confirming this. -Supported platforms: - -- Azure Repos -- Bitbucket Server -- GitHub -- GitLab - Supported package management tools: - Go @@ -112,89 +118,89 @@ Supported package management tools: - Poetry - Yarn 2 -#### 🕵️‍♀️ How does Pull Request scanning work? +#### How to uuse Pull Request scanning? -
- Azure Repos +
+ Azure Repos -After you create a new pull request, Frogbot will automatically scan it. + After you create a new pull request, Frogbot will automatically scan it. -> **_NOTE:_** The scan output will include only new vulnerabilities added by the pull request. -> Vulnerabilities that aren't new, and existed in the code before the pull request was created, will not be included in -> the -> report. In order to include all the vulnerabilities in the report, including older ones that weren't added by this -> PR, use the includeAllVulnerabilities parameter in the frogbot-config.yml file. + > **_NOTE:_** The scan output will include only new vulnerabilities added by the pull request. + > Vulnerabilities that aren't new, and existed in the code before the pull request was created, will not be included in + > the + > report. In order to include all the vulnerabilities in the report, including older ones that weren't added by this + > PR, use the includeAllVulnerabilities parameter in the frogbot-config.yml file. -The Frogbot Azure Repos scan workflow is: + The Frogbot Azure Repos scan workflow is: -1. The developer opens a pull request. -2. Frogbot scans the pull request and adds a comment with the scan results. -3. Frogbot can be triggered again following new commits, by adding a comment with the `rescan` text. + 1. The developer opens a pull request. + 2. Frogbot scans the pull request and adds a comment with the scan results. + 3. Frogbot can be triggered again following new commits, by adding a comment with the `rescan` text. -
+
-
- Bitbucket Server +
+ Bitbucket Server -After you create a new pull request, Frogbot will automatically scan it. + After you create a new pull request, Frogbot will automatically scan it. -> **_NOTE:_** The scan output will include only new vulnerabilities added by the pull request. -> Vulnerabilities that aren't new, and existed in the code before the pull request was created, will not be included in -> the -> report. In order to include all of the vulnerabilities in the report, including older ones that weren't added by this -> PR, use the includeAllVulnerabilities parameter in the frogbot-config.yml file. + > **_NOTE:_** The scan output will include only new vulnerabilities added by the pull request. + > Vulnerabilities that aren't new, and existed in the code before the pull request was created, will not be included in + > the + > report. In order to include all of the vulnerabilities in the report, including older ones that weren't added by this + > PR, use the includeAllVulnerabilities parameter in the frogbot-config.yml file. -The Frogbot scan on Bitbucket Server workflow: + The Frogbot scan on Bitbucket Server workflow: -1. The developer opens a pull request. -2. Frogbot scans the pull request and adds a comment with the scan results. -3. Frogbot can be triggered again following new commits, by adding a comment with the `rescan` text. + 1. The developer opens a pull request. + 2. Frogbot scans the pull request and adds a comment with the scan results. + 3. Frogbot can be triggered again following new commits, by adding a comment with the `rescan` text. -
+
-
- GitHub +
+ GitHub -After you create a new pull request, the maintainer of the Git repository can trigger Frogbot to scan the pull request from the pull request UI. + After you create a new pull request, the maintainer of the Git repository can trigger Frogbot to scan the pull request from the pull request UI. -> **_NOTE:_** The scan output will include only new vulnerabilities added by the pull request. -> Vulnerabilities that aren't new, and existed in the code before the pull request was created, will not be included in -> the -> report. In order to include all of the vulnerabilities in the report, including older ones that weren't added by this -> PR, use the includeAllVulnerabilities parameter in the frogbot-config.yml file. + > **_NOTE:_** The scan output will include only new vulnerabilities added by the pull request. + > Vulnerabilities that aren't new, and existed in the code before the pull request was created, will not be included in + > the + > report. In order to include all of the vulnerabilities in the report, including older ones that weren't added by this + > PR, use the includeAllVulnerabilities parameter in the frogbot-config.yml file. -The Frogbot GitHub scan workflow is: + The Frogbot GitHub scan workflow is: -1. The developer opens a pull request. -2. The Frogbot workflow automatically gets triggered and a [GitHub environment](https://docs.github.com/en/actions/deployment/targeting-different-environments/using-environments-for-deployment#creating-an-environment) named `frogbot` becomes pending for the maintainer's approval. + 1. The developer opens a pull request. + 2. The Frogbot workflow automatically gets triggered and a [GitHub environment](https://docs.github.com/en/actions/deployment/targeting-different-environments/using-environments-for-deployment#creating-an-environment) named `frogbot` becomes pending for the maintainer's approval. -[![](./images/github-pending-deployment.png)](#running-frogbot-on-github) + [![](./images/github-pending-deployment.png)](#running-frogbot-on-github) -3. The maintainer of the repository reviews the pull request and approves the scan: [![](./images/github-deployment.gif)](#running-frogbot-on-github) -4. Frogbot can be triggered again following new commits, by repeating steps 2 and 3. + 3. The maintainer of the repository reviews the pull request and approves the scan: [![](./images/github-deployment.gif)](#running-frogbot-on-github) + 4. Frogbot can be triggered again following new commits, by repeating steps 2 and 3. -
+
-
- GitLab +
+ GitLab -After you create a new merge request, the maintainer of the Git repository can trigger Frogbot to scan the merge request from the merge request UI. + After you create a new merge request, the maintainer of the Git repository can trigger Frogbot to scan the merge request from the merge request UI. -> **_NOTE:_** The scan output will include only new vulnerabilities added by the merge request. -> Vulnerabilities that aren't new, and existed in the code before the merge request was created, will not be included in -> the -> report. In order to include all of the vulnerabilities in the report, including older ones that weren't added by this -> merge request, use the includeAllVulnerabilities parameter in the frogbot-config.yml file. + > **_NOTE:_** The scan output will include only new vulnerabilities added by the merge request. + > Vulnerabilities that aren't new, and existed in the code before the merge request was created, will not be included in + > the + > report. In order to include all of the vulnerabilities in the report, including older ones that weren't added by this + > merge request, use the includeAllVulnerabilities parameter in the frogbot-config.yml file. -The Frogbot GitLab flow is as follows: + The Frogbot GitLab flow is as follows: -1. The developer opens a merge request. -2. The maintainer of the repository reviews the merge request and approves the scan by triggering the manual _frogbot-scan_ job. -3. Frogbot is then triggered by the job, it scans the merge request, and adds a comment with the scan results. -4. Frogbot can be triggered again following new commits, by triggering the _frogbot-scan_ job again. - [GitLab CI Run Button](./images/gitlab-run-button.png) + 1. The developer opens a merge request. + 2. The maintainer of the repository reviews the merge request and approves the scan by triggering the manual _frogbot-scan_ job. + 3. Frogbot is then triggered by the job, it scans the merge request, and adds a comment with the scan results. + 4. Frogbot can be triggered again following new commits, by triggering the _frogbot-scan_ job again. + [GitLab CI Run Button](./images/gitlab-run-button.png) -
+
#### 👮 Security note for pull requests scanning diff --git a/docs/install-bitbucket-server.md b/docs/install-bitbucket-server.md index d7e782419..12d51fe27 100644 --- a/docs/install-bitbucket-server.md +++ b/docs/install-bitbucket-server.md @@ -13,193 +13,194 @@ named **jfrogPlatform**. * Save your Bitbucket access token in a [Bitbucket Server Integration](https://www.jfrog.com/confluence/display/JFROG/Bitbucket+Server+Integration) named **gitIntegration**. - * Create a **pipelines.yml** file using one of the available [templates](templates/jfrog-pipelines) and push the file to your Frogbot Management Git repository under a directory named `.jfrog-pipelines`. + * Create a **pipelines.yml** file using one of the available [templates](templates/jfrog-pipelines) and push the file into one of your Git repositories, under a directory named `.jfrog-pipelines`. * In the **pipelines.yml**, make sure to set values for all the mandatory variables. * In the **pipelines.yml**, if you're using a Windows agent, modify the code inside the onExecute sections as described in the template comments. **Important** - Make sure all the build tools that are used to build the project are installed on the build agent. - -
- Install Frogbot Using Jenkins +
+
+ Install Frogbot Using Jenkins - Make sure you have the connection details of your JFrog environment. - Save the JFrog connection details as Credentials in Jenkins with the following Credential IDs: **JF_URL**, **JF_USER** and **JF_PASSWORD** (You can also use **JF_XRAY_URL** and **JF_ARTIFACTORY_URL** instead of **JF_URL** and **JF_ACCESS_TOKEN** instead of **JF_USER** and **JF_PASSWORD**). - Save your Bitbucket access token as a Credential in Jenkins with the `FROGBOT_GIT_TOKEN` Credential ID. - - Create a Jenkinsfile with the below content under the root of your **Frogbot Management Repository**. + - Create a Jenkinsfile with the below template content, and push it to root of one of your Git repositories. - In the Jenkinsfile, set the values of all the mandatory variables. - In the Jenkinsfile, modify the code inside the `Download Frogbot` and `Scan Pull Requests` according to the Jenkins agent operating system. - - Create a Pipeline job in Jenkins pointing to the Jenkinsfile in your **Frogbot Management Repository**. - - ```groovy - // Run the job once an hour - CRON_SETTINGS = '''* */1 * * *''' - - pipeline { - agent any - - triggers { - cron(CRON_SETTINGS) - } - - environment { - // [Mandatory] - // JFrog platform URL (This functionality requires version 3.29.0 or above of Xray) - JF_URL= credentials("JF_URL") - - // [Mandatory if JF_USER and JF_PASSWORD are not provided] - // JFrog access token with 'read' permissions for Xray - JF_ACCESS_TOKEN= credentials("JF_ACCESS_TOKEN") - - // [Mandatory if JF_ACCESS_TOKEN is not provided] - // JFrog user and password with 'read' permissions for Xray - // JF_USER= credentials("JF_USER") - // JF_PASSWORD= credentials("JF_PASSWORD") - - // [Mandatory] - // Bitbucket access token with the write repository permissions - JF_GIT_TOKEN= credentials("FROGBOT_GIT_TOKEN") - JF_GIT_PROVIDER= "bitbucketServer" - - // [Mandatory] - // Username of the Bitbucket account - JF_GIT_USERNAME= "" - - // [Mandatory] - // Bitbucket project namespace - JF_GIT_OWNER= "" - - // [Mandatory] - // API endpoint to Bitbucket server - JF_GIT_API_ENDPOINT= "" - - // [Optional] - // If the machine that runs Frogbot has no access to the internet, set the name of a remote repository - // in Artifactory, which proxies https://releases.jfrog.io - // The 'frogbot' executable and other tools it needs will be downloaded through this repository. - // JF_RELEASES_REPO= "" - - - - - ////////////////////////////////////////////////////////////////////////// - // If your project uses a 'frogbot-config.yml' file, you can define // - // the following variables inside the file, instead of here. // - ////////////////////////////////////////////////////////////////////////// - - // [Mandatory if the two conditions below are met] - // 1. The project uses yarn 2, NuGet or .NET to download its dependencies - // 2. The `installCommand` variable isn't set in your frogbot-config.yml file. - // - // The command that installs the project dependencies (e.g "nuget restore") - JF_INSTALL_DEPS_CMD= "" - - // [Optional, default: "."] - // Relative path to the root of the project in the Git repository - // JF_WORKING_DIR= path/to/project/dir - - // [Optional] - // Xray Watches. Learn more about them here: https://www.jfrog.com/confluence/display/JFROG/Configuring+Xray+Watches - // JF_WATCHES= ,... - - // [Optional] - // JFrog project. Learn more about it here: https://www.jfrog.com/confluence/display/JFROG/Projects - // JF_PROJECT= - - // [Optional, default: "FALSE"] - // Displays all existing vulnerabilities, including the ones that were added by the pull request. - // JF_INCLUDE_ALL_VULNERABILITIES= "TRUE" - - // [Optional, default: "TRUE"] - // Fails the Frogbot task if any security issue is found. - // JF_FAIL= "FALSE" - - // [Optional, default: "TRUE"] - // Relative path to a Pip requirements.txt file. If not set, the python project's dependencies are determined and scanned using the project setup.py file. - // JF_REQUIREMENTS_FILE= "" - - // [Optional, Default: "TRUE"] - // Use Gradle wrapper. - // JF_USE_WRAPPER= "FALSE" - - // [Optional] - // Frogbot will download the project dependencies if they're not cached locally. To download the - // dependencies from a virtual repository in Artifactory, set the name of of the repository. There's no - // need to set this value, if it is set in the frogbot-config.yml file. - // JF_DEPS_REPO= "" - - // [Optional] - // Template for the branch name generated by Frogbot when creating pull requests with fixes. - // The template must include ${BRANCH_NAME_HASH}, to ensure that the generated branch name is unique. - // The template can optionally include the ${IMPACTED_PACKAGE} and ${FIX_VERSION} variables. - // JF_BRANCH_NAME_TEMPLATE= "frogbot-${IMPACTED_PACKAGE}-${BRANCH_NAME_HASH}" - - // [Optional] - // Template for the commit message generated by Frogbot when creating pull requests with fixes - // The template can optionally include the ${IMPACTED_PACKAGE} and ${FIX_VERSION} variables. - // JF_COMMIT_MESSAGE_TEMPLATE= "Upgrade ${IMPACTED_PACKAGE} to ${FIX_VERSION}" + - Create a Pipeline job in Jenkins pointing to the Jenkinsfile in your Git repository. - // [Optional] - // Template for the pull request title generated by Frogbot when creating pull requests with fixes. - // The template can optionally include the ${IMPACTED_PACKAGE} and ${FIX_VERSION} variables. - // JF_PULL_REQUEST_TITLE_TEMPLATE= "[🐸 Frogbot] Upgrade ${IMPACTED_PACKAGE} to to ${FIX_VERSION}" - - // [Optional, Default: "FALSE"] - // If TRUE, Frogbot creates a single pull request with all the fixes. - // If FALSE, Frogbot creates a separate pull request for each fix. - // JF_GIT_AGGREGATE_FIXES= "FALSE" +
+ Template - // [Optional, Default: "FALSE"] - // Handle vulnerabilities with fix versions only - // JF_FIXABLE_ONLY= "TRUE" - - // [Optional] - // Set the minimum severity for vulnerabilities that should be fixed and commented on in pull requests - // The following values are accepted: Low, Medium, High or Critical - // JF_MIN_SEVERITY= "" - } + ```groovy + // Run the job once an hour + CRON_SETTINGS = '''* */1 * * *''' - stages { - stage('Download Frogbot') { - steps { - // For Linux / MacOS runner: - sh """ curl -fLg "https://releases.jfrog.io/artifactory/frogbot/v2/[RELEASE]/getFrogbot.sh" | sh""" + pipeline { + agent any - // For Windows runner: - // powershell """iwr https://releases.jfrog.io/artifactory/frogbot/v2/[RELEASE]/frogbot-windows-amd64/frogbot.exe -OutFile .\frogbot.exe""" - } - } + triggers { + cron(CRON_SETTINGS) + } - stage('Scan Pull Requests') { - steps { - sh "./frogbot scan-pull-requests" + environment { + // [Mandatory] + // JFrog platform URL (This functionality requires version 3.29.0 or above of Xray) + JF_URL= credentials("JF_URL") + + // [Mandatory if JF_USER and JF_PASSWORD are not provided] + // JFrog access token with 'read' permissions for Xray + JF_ACCESS_TOKEN= credentials("JF_ACCESS_TOKEN") + + // [Mandatory if JF_ACCESS_TOKEN is not provided] + // JFrog user and password with 'read' permissions for Xray + // JF_USER= credentials("JF_USER") + // JF_PASSWORD= credentials("JF_PASSWORD") + + // [Mandatory] + // Bitbucket access token with the write repository permissions + JF_GIT_TOKEN= credentials("FROGBOT_GIT_TOKEN") + JF_GIT_PROVIDER= "bitbucketServer" + + // [Mandatory] + // Username of the Bitbucket account + JF_GIT_USERNAME= "" + + // [Mandatory] + // Bitbucket project namespace + JF_GIT_OWNER= "" + + // [Mandatory] + // API endpoint to Bitbucket server + JF_GIT_API_ENDPOINT= "" + + // [Optional] + // If the machine that runs Frogbot has no access to the internet, set the name of a remote repository + // in Artifactory, which proxies https://releases.jfrog.io + // The 'frogbot' executable and other tools it needs will be downloaded through this repository. + // JF_RELEASES_REPO= "" + + + + + ////////////////////////////////////////////////////////////////////////// + // If your project uses a 'frogbot-config.yml' file, you can define // + // the following variables inside the file, instead of here. // + ////////////////////////////////////////////////////////////////////////// + + // [Mandatory if the two conditions below are met] + // 1. The project uses yarn 2, NuGet or .NET to download its dependencies + // 2. The `installCommand` variable isn't set in your frogbot-config.yml file. + // + // The command that installs the project dependencies (e.g "nuget restore") + JF_INSTALL_DEPS_CMD= "" + + // [Optional, default: "."] + // Relative path to the root of the project in the Git repository + // JF_WORKING_DIR= path/to/project/dir + + // [Optional] + // Xray Watches. Learn more about them here: https://www.jfrog.com/confluence/display/JFROG/Configuring+Xray+Watches + // JF_WATCHES= ,... + + // [Optional] + // JFrog project. Learn more about it here: https://www.jfrog.com/confluence/display/JFROG/Projects + // JF_PROJECT= + + // [Optional, default: "FALSE"] + // Displays all existing vulnerabilities, including the ones that were added by the pull request. + // JF_INCLUDE_ALL_VULNERABILITIES= "TRUE" + + // [Optional, default: "TRUE"] + // Fails the Frogbot task if any security issue is found. + // JF_FAIL= "FALSE" - // For Windows runner: - // powershell """.\frogbot.exe scan-pull-requests""" - } - } + // [Optional, default: "TRUE"] + // Relative path to a Pip requirements.txt file. If not set, the python project's dependencies are determined and scanned using the project setup.py file. + // JF_REQUIREMENTS_FILE= "" + + // [Optional, Default: "TRUE"] + // Use Gradle wrapper. + // JF_USE_WRAPPER= "FALSE" + + // [Optional] + // Frogbot will download the project dependencies if they're not cached locally. To download the + // dependencies from a virtual repository in Artifactory, set the name of of the repository. There's no + // need to set this value, if it is set in the frogbot-config.yml file. + // JF_DEPS_REPO= "" + + // [Optional] + // Template for the branch name generated by Frogbot when creating pull requests with fixes. + // The template must include ${BRANCH_NAME_HASH}, to ensure that the generated branch name is unique. + // The template can optionally include the ${IMPACTED_PACKAGE} and ${FIX_VERSION} variables. + // JF_BRANCH_NAME_TEMPLATE= "frogbot-${IMPACTED_PACKAGE}-${BRANCH_NAME_HASH}" + + // [Optional] + // Template for the commit message generated by Frogbot when creating pull requests with fixes + // The template can optionally include the ${IMPACTED_PACKAGE} and ${FIX_VERSION} variables. + // JF_COMMIT_MESSAGE_TEMPLATE= "Upgrade ${IMPACTED_PACKAGE} to ${FIX_VERSION}" + + // [Optional] + // Template for the pull request title generated by Frogbot when creating pull requests with fixes. + // The template can optionally include the ${IMPACTED_PACKAGE} and ${FIX_VERSION} variables. + // JF_PULL_REQUEST_TITLE_TEMPLATE= "[🐸 Frogbot] Upgrade ${IMPACTED_PACKAGE} to to ${FIX_VERSION}" + + // [Optional, Default: "FALSE"] + // If TRUE, Frogbot creates a single pull request with all the fixes. + // If FALSE, Frogbot creates a separate pull request for each fix. + // JF_GIT_AGGREGATE_FIXES= "FALSE" + + // [Optional, Default: "FALSE"] + // Handle vulnerabilities with fix versions only + // JF_FIXABLE_ONLY= "TRUE" - stage('Scan and Fix Repos') { + // [Optional] + // Set the minimum severity for vulnerabilities that should be fixed and commented on in pull requests + // The following values are accepted: Low, Medium, High or Critical + // JF_MIN_SEVERITY= "" + } + + stages { + stage('Download Frogbot') { steps { - sh "./frogbot scan-and-fix-repos" + // For Linux / MacOS runner: + sh """ curl -fLg "https://releases.jfrog.io/artifactory/frogbot/v2/[RELEASE]/getFrogbot.sh" | sh""" + + // For Windows runner: + // powershell """iwr https://releases.jfrog.io/artifactory/frogbot/v2/[RELEASE]/frogbot-windows-amd64/frogbot.exe -OutFile .\frogbot.exe""" + } + } + + stage('Scan Pull Requests') { + steps { + sh "./frogbot scan-pull-requests" + + // For Windows runner: + // powershell """.\frogbot.exe scan-pull-requests""" + } + } + + stage('Scan and Fix Repos') { + steps { + sh "./frogbot scan-and-fix-repos" + + // For Windows runner: + // powershell """.\frogbot.exe scan-and-fix-repos""" + } + } + } + } + ``` - // For Windows runner: - // powershell """.\frogbot.exe scan-and-fix-repos""" - } - } - } - } - ``` - **Important** - - Make sure that either **JF_USER** and **JF_PASSWORD** or **JF_ACCESS_TOKEN** are set in the Jenkinsfile, but not both. - - Make sure that all the build tools that are used to build the project are installed on the Jenkins agent. - + - Make sure that either **JF_USER** and **JF_PASSWORD** or **JF_ACCESS_TOKEN** are set in the Jenkinsfile, but not both. + - Make sure that all the build tools that are used to build the project are installed on the Jenkins agent.
- -
+ From c625b4f86e03a9db621a06e105b9050d3a2e377d Mon Sep 17 00:00:00 2001 From: Eyal Ben Moshe Date: Thu, 25 May 2023 16:24:35 +0300 Subject: [PATCH 5/6] Docs updates (#348) --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index acd4c7e5e..3b3cf3b48 100644 --- a/README.md +++ b/README.md @@ -42,7 +42,7 @@ Frogbot supports the following Git providers: - GitHub - GitLab. -### What's needed for the set up? +### What's needed for the setup? - Frogbot uses a JFrog environment to scan your Git repositories. If you don't have a JFrog environment, you can set up one for free, and use it with no limits. - Frogbot also requires a runtime environment for the scanning. The following environments are supported: From aa1fdf7f36733df1fbd81e6b1693b3cb708121a3 Mon Sep 17 00:00:00 2001 From: Eyal Ben Moshe Date: Fri, 26 May 2023 16:02:06 +0300 Subject: [PATCH 6/6] Upgrade jfrog-cli-core to v2.34.5 (#351) --- go.mod | 10 +++++----- go.sum | 4 ++-- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/go.mod b/go.mod index eb19bb6f7..d2ca43d2c 100644 --- a/go.mod +++ b/go.mod @@ -8,7 +8,7 @@ require ( github.com/jfrog/build-info-go v1.9.6 github.com/jfrog/froggit-go v1.7.3 github.com/jfrog/gofrog v1.3.0 - github.com/jfrog/jfrog-cli-core/v2 v2.34.4 + github.com/jfrog/jfrog-cli-core/v2 v2.34.5 github.com/jfrog/jfrog-client-go v1.29.1 github.com/mholt/archiver/v3 v3.5.1 github.com/pkg/errors v0.9.1 @@ -112,10 +112,10 @@ require ( gopkg.in/warnings.v0 v0.1.2 // indirect ) -replace github.com/jfrog/jfrog-cli-core/v2 => github.com/jfrog/jfrog-cli-core/v2 v2.31.1-0.20230524155422-b3ac4175049a +// replace github.com/jfrog/jfrog-cli-core/v2 => github.com/jfrog/jfrog-cli-core/v2 v2.31.1-0.20230524155422-b3ac4175049a -//replace github.com/jfrog/build-info-go => github.com/jfrog/build-info-go v1.8.9-0.20230518114837-fe6a826d5001 +// replace github.com/jfrog/build-info-go => github.com/jfrog/build-info-go v1.8.9-0.20230518114837-fe6a826d5001 -//replace github.com/jfrog/jfrog-client-go => github.com/jfrog/jfrog-client-go v1.28.1-0.20230430083747-590ae14f9dca +// replace github.com/jfrog/jfrog-client-go => github.com/jfrog/jfrog-client-go v1.28.1-0.20230430083747-590ae14f9dca -//replace github.com/jfrog/gofrog => github.com/jfrog/gofrog v1.2.6-0.20230418122323-2bf299dd6d27 +// replace github.com/jfrog/gofrog => github.com/jfrog/gofrog v1.2.6-0.20230418122323-2bf299dd6d27 diff --git a/go.sum b/go.sum index d13c23471..a850b30c1 100644 --- a/go.sum +++ b/go.sum @@ -230,8 +230,8 @@ github.com/jfrog/froggit-go v1.7.3 h1:ZOXbFsdmtM6+uDbo1U+elV7sPZ4EN6grDck9aHqKyn github.com/jfrog/froggit-go v1.7.3/go.mod h1:xfsfQXzSaAM04RV9IyU5heBiRrsm2oS6rFCfEofQr6U= github.com/jfrog/gofrog v1.3.0 h1:o4zgsBZE4QyDbz2M7D4K6fXPTBJht+8lE87mS9bw7Gk= github.com/jfrog/gofrog v1.3.0/go.mod h1:IFMc+V/yf7rA5WZ74CSbXe+Lgf0iApEQLxRZVzKRUR0= -github.com/jfrog/jfrog-cli-core/v2 v2.31.1-0.20230524155422-b3ac4175049a h1:314Ti3MclykCssxZvJjbLnBZgaNknOw8hEaWmMLAt+0= -github.com/jfrog/jfrog-cli-core/v2 v2.31.1-0.20230524155422-b3ac4175049a/go.mod h1:htVCrvgB2P3ROJNQEPs2LptVTQ33l8349q9t/zD/hAM= +github.com/jfrog/jfrog-cli-core/v2 v2.34.5 h1:CqB7tGPazR9z7diTQJVpW/xvTUykM69NgJvrCqZZB3c= +github.com/jfrog/jfrog-cli-core/v2 v2.34.5/go.mod h1:htVCrvgB2P3ROJNQEPs2LptVTQ33l8349q9t/zD/hAM= github.com/jfrog/jfrog-client-go v1.29.1 h1:R5NyZ6qbroY8uG6vWX/5nLjATMo8OMOhyVd3GcejFDI= github.com/jfrog/jfrog-client-go v1.29.1/go.mod h1:nGUoz5Qi9kTP0VfkvOVJ3nudsD3dq3y/d0sLjlkpnrI= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=