Remove gradle extractor and use gradle-dep-tree (#283)

commands/createfixpullrequests.go

@@ -68,6 +68,7 @@ func (cfp *CreateFixPullRequestsCmd) scanAndFixRepository(repository *utils.Frog
 		Client:                   client,
 		FailOnInstallationErrors: *repository.FailOnSecurityIssues,
 		Branch:                   branch,
+		ReleasesRepo:             repository.JfrogReleasesRepo,
 	}
 	for _, project := range repository.Projects {
 		cfp.details.Project = project

commands/scanandfixrepos_test.go

@@ -55,7 +55,7 @@ func TestScanAndFixRepos(t *testing.T) {
 	defer cleanUp()
 
 	createReposGitEnvironment(t, tmpDir, port, testRepositories...)
-	configAggregator, err := utils.NewConfigAggregatorFromFile(configData, gitTestParams, &serverParams)
+	configAggregator, err := utils.NewConfigAggregatorFromFile(configData, gitTestParams, &serverParams, "")
 	assert.NoError(t, err)
 
 	var cmd = ScanAndFixRepositories{dryRun: true, dryRunRepoPath: filepath.Join("testdata", "scanandfixrepos")}

commands/scanpullrequest.go

@@ -85,6 +85,7 @@ func auditPullRequest(repoConfig *utils.FrogbotRepoConfig, client vcsclient.VcsC
 			Git:                      &repoConfig.Git,
 			FailOnInstallationErrors: false,
 			Branch:                   repoConfig.Branches[0],
+			ReleasesRepo:             repoConfig.JfrogReleasesRepo,
 		}
 		currentScan, isMultipleRoot, err := auditSource(scanSetup)
 		if err != nil {
@@ -271,6 +272,7 @@ func runInstallAndAudit(scanSetup *utils.ScanDetails, workDirs ...string) (resul
 		SetRequirementsFile(scanSetup.PipRequirementsFile).
 		SetWorkingDirs(workDirs).
 		SetDepsRepo(scanSetup.Repository).
+		SetReleasesRepo(scanSetup.ReleasesRepo).
 		SetIgnoreConfigFile(true)
 	results, isMultipleRoot, err = audit.GenericAudit(auditParams)
 	if err != nil {

commands/scanpullrequest_test.go

@@ -554,7 +554,7 @@ func prepareConfigAndClient(t *testing.T, configPath string, server *httptest.Se
 
 	configData, err := utils.ReadConfigFromFileSystem(configPath)
 	assert.NoError(t, err)
-	configAggregator, err := utils.NewConfigAggregatorFromFile(configData, git, &serverParams)
+	configAggregator, err := utils.NewConfigAggregatorFromFile(configData, git, &serverParams, "")
 	assert.NoError(t, err)
 
 	client, err := vcsclient.NewClientBuilder(vcsutils.GitLab).ApiEndpoint(server.URL).Token("123456").Build()

commands/utils/extractors.go

@@ -12,7 +12,7 @@ import (
 	"github.com/jfrog/jfrog-client-go/utils/log"
 )
 
-var extractorsRepositoryPath = filepath.Join("artifactory", "oss-release-local")
+var extractorsRepositoryPath = "artifactory/oss-release-local"
 
 // extractorDetails holds the relevant details to download the build-info extractors.
 // Build Info is Artifactory's open integration layer for the CI servers and build tools.
@@ -33,55 +33,38 @@ func (ed *extractorDetails) downloadFromPath() string {
 }
 
 // downloadExtractorsFromRemoteIfNeeded downloads build-info-extractors from a remote repository, if they do not yet exist on the file system.
-func downloadExtractorsFromRemoteIfNeeded(server *config.ServerDetails, extractorsLocalPath string) (err error) {
-	var releasesRepo string
+func downloadExtractorsFromRemoteIfNeeded(server *config.ServerDetails, extractorsLocalPath string) (releasesRepo string, err error) {
 	if releasesRepo = getTrimmedEnv(jfrogReleasesRepoEnv); releasesRepo == "" {
-		return nil
+		return
 	}
 	// Download extractors if remote repo environment variable is set
 	log.Info("Checking whether the build-info extractors exist locally")
 	if extractorsLocalPath == "" {
 		extractorsLocalPath, err = config.GetJfrogDependenciesPath()
 		if err != nil {
-			return err
+			return
 		}
 	}
 	mavenExtractorLocalPath := filepath.Join(extractorsLocalPath, "maven", build.MavenExtractorDependencyVersion)
-	gradleExtractorLocalPath := filepath.Join(extractorsLocalPath, "gradle", build.GradleExtractorDependencyVersion)
-	extractors := []extractorDetails{
-		{
-			extractorType: coreutils.Maven.ToString(),
-			localPath:     mavenExtractorLocalPath,
-			fileName:      fmt.Sprintf(build.MavenExtractorFileName, build.MavenExtractorDependencyVersion),
-			remotePath:    fmt.Sprintf(build.MavenExtractorRemotePath, build.MavenExtractorDependencyVersion),
-		},
-		{
-			extractorType: coreutils.Gradle.ToString(),
-			localPath:     gradleExtractorLocalPath,
-			fileName:      fmt.Sprintf(build.GradleExtractorFileName, build.GradleExtractorDependencyVersion),
-			remotePath:    fmt.Sprintf(build.GradleExtractorRemotePath, build.GradleExtractorDependencyVersion),
-		},
+	mavenExtractor := extractorDetails{
+		extractorType: coreutils.Maven.ToString(),
+		localPath:     mavenExtractorLocalPath,
+		fileName:      fmt.Sprintf(build.MavenExtractorFileName, build.MavenExtractorDependencyVersion),
+		remotePath:    fmt.Sprintf(build.MavenExtractorRemotePath, build.MavenExtractorDependencyVersion),
 	}
-	return downloadExtractors(releasesRepo, server, extractors...)
+	return releasesRepo, downloadExtractor(releasesRepo, server, mavenExtractor)
 }
 
-func downloadExtractors(remoteRepoName string, server *config.ServerDetails, extractors ...extractorDetails) (err error) {
-	for _, extractor := range extractors {
-		var alreadyExist bool
-		if alreadyExist, err = fileutils.IsDirExists(extractor.localPath, false); alreadyExist {
-			log.Debug(extractor.extractorType, "extractor already exists, no download necessary")
-			continue
-		}
-		if err != nil {
-			return err
-		}
-		log.Info("Downloading", extractor.extractorType, "extractor to path:", extractor.localPath)
-		remoteServer := getRemoteServer(server, remoteRepoName)
-		if err = utils.DownloadExtractor(remoteServer, extractor.downloadFromPath(), extractor.downloadToPath()); err != nil {
-			return err
-		}
+func downloadExtractor(remoteRepoName string, server *config.ServerDetails, extractor extractorDetails) (err error) {
+	var alreadyExist bool
+	if alreadyExist, err = fileutils.IsDirExists(extractor.localPath, false); alreadyExist {
+		log.Debug(extractor.extractorType, "extractor already exists, no download required")
+		return
 	}
-	return
+
+	log.Info("Downloading", extractor.extractorType, "extractor to path:", extractor.localPath)
+	remoteServer := getRemoteServer(server, remoteRepoName)
+	return utils.DownloadExtractor(remoteServer, extractor.downloadFromPath(), extractor.downloadToPath())
 }
 
 func getRemoteServer(server *config.ServerDetails, remoteName string) *config.ServerDetails {

commands/utils/extractors_test.go

@@ -14,10 +14,12 @@ import (
 
 func TestDownloadExtractorsFromRemoteIfNeeded(t *testing.T) {
 	serverDetails := &config.ServerDetails{
-		AccessToken: "eyJ0eXAiOiJKV1QifQ.eyJzdWIiOiJmYWtlXC91c2Vyc1wvdGVzdCJ9.MTIzNDU2Nzg5MA",
+		AccessToken: "eyJ0eXAiOiJKV1QifQ.eyJzdWIiOiJmYWtlXC91c2Vy2323c1wvdGVzdCJ9.MTIzNDU2Nzg5MA",
 	}
 	assert.NoError(t, os.Setenv(jfrogReleasesRepoEnv, "remote-repo"))
-	defer assert.NoError(t, os.Unsetenv(jfrogReleasesRepoEnv))
+	defer func() {
+		assert.NoError(t, os.Unsetenv(jfrogReleasesRepoEnv))
+	}()
 	tmpDir, err := fileutils.CreateTempDir()
 	assert.NoError(t, err)
 	restoreDir, err := Chdir(tmpDir)
@@ -27,9 +29,13 @@ func TestDownloadExtractorsFromRemoteIfNeeded(t *testing.T) {
 		assert.NoError(t, fileutils.RemoveTempDir(tmpDir))
 	}()
 	testServer := httptest.NewServer(createRemoteArtifactoryHandler())
-	defer testServer.Close()
+	defer func() {
+		testServer.Close()
+	}()
 	serverDetails.ArtifactoryUrl = testServer.URL + "/artifactory/"
-	assert.NoError(t, downloadExtractorsFromRemoteIfNeeded(serverDetails, tmpDir))
+	releasesRepo, err := downloadExtractorsFromRemoteIfNeeded(serverDetails, tmpDir)
+	assert.NoError(t, err)
+	assert.Equal(t, "remote-repo", releasesRepo)
 }
 
 // Create HTTP handler to mock remote artifactory server
@@ -38,10 +44,7 @@ func createRemoteArtifactoryHandler() http.HandlerFunc {
 		expectedMavenUri := fmt.Sprintf("/artifactory/remote-repo/artifactory/oss-release-local/%s/%s",
 			fmt.Sprintf(build.MavenExtractorRemotePath, build.MavenExtractorDependencyVersion),
 			fmt.Sprintf(build.MavenExtractorFileName, build.MavenExtractorDependencyVersion))
-		expectedGradleUri := fmt.Sprintf("/artifactory/remote-repo/artifactory/oss-release-local/%s/%s",
-			fmt.Sprintf(build.GradleExtractorRemotePath, build.GradleExtractorDependencyVersion),
-			fmt.Sprintf(build.GradleExtractorFileName, build.GradleExtractorDependencyVersion))
-		if r.RequestURI == expectedMavenUri || r.RequestURI == expectedGradleUri {
+		if r.RequestURI == expectedMavenUri {
 			w.WriteHeader(http.StatusOK)
 			return
 		}

commands/utils/params.go

@@ -99,6 +99,7 @@ type Scan struct {
 	IncludeAllVulnerabilities bool      `yaml:"includeAllVulnerabilities,omitempty"`
 	FailOnSecurityIssues      *bool     `yaml:"failOnSecurityIssues,omitempty"`
 	Projects                  []Project `yaml:"projects,omitempty"`
+	JfrogReleasesRepo         string
 }
 
 func (s *Scan) setDefaultsIfNeeded() *Scan {
@@ -167,14 +168,15 @@ func GetFrogbotUtils() (frogbotUtils *FrogbotUtils, err error) {
 
 // getConfigAggregator returns a FrogbotConfigAggregator based on frogbot-config.yml and environment variables.
 func getConfigAggregator(client vcsclient.VcsClient, server *coreconfig.ServerDetails, gitParams *Git) (FrogbotConfigAggregator, error) {
-	if err := downloadExtractorsFromRemoteIfNeeded(server, ""); err != nil {
+	releasesRepo, err := downloadExtractorsFromRemoteIfNeeded(server, "")
+	if err != nil {
 		return nil, err
 	}
 	configFileContent, err := getConfigFileContent(client)
 	// If there is a missing configuration file error, try to generate an environment variable-based config aggregator.
 	if _, missingConfigErr := err.(*ErrMissingConfig); missingConfigErr {
 		log.Debug("Unable to retrieve", FrogbotConfigFile, err.Error())
-		configAggregator, err := newConfigAggregatorFromEnv(gitParams, server)
+		configAggregator, err := newConfigAggregatorFromEnv(gitParams, server, releasesRepo)
 		if err != nil {
 			return nil, err
 		}
@@ -183,7 +185,7 @@ func getConfigAggregator(client vcsclient.VcsClient, server *coreconfig.ServerDe
 		return nil, err
 	}
 
-	return NewConfigAggregatorFromFile(configFileContent, gitParams, server)
+	return NewConfigAggregatorFromFile(configFileContent, gitParams, server, releasesRepo)
 }
 
 // The getConfigFileContent function retrieves the frogbot-config.yml file content.
@@ -205,7 +207,7 @@ func getConfigFileContent(client vcsclient.VcsClient) (configFileContent []byte,
 }
 
 // NewConfigAggregatorFromFile receive a frogbot-config.yml file content along with the Git and ServerDetails parameters, and returns a FrogbotConfigAggregator instance with all the default and necessary fields.
-func NewConfigAggregatorFromFile(configFileContent []byte, gitParams *Git, server *coreconfig.ServerDetails) (result FrogbotConfigAggregator, err error) {
+func NewConfigAggregatorFromFile(configFileContent []byte, gitParams *Git, server *coreconfig.ServerDetails, releasesRepo string) (result FrogbotConfigAggregator, err error) {
 	// Unmarshal the frogbot-config.yml file
 	result, err = result.UnmarshalYaml(configFileContent)
 	if err != nil {
@@ -217,6 +219,7 @@ func NewConfigAggregatorFromFile(configFileContent []byte, gitParams *Git, serve
 		if result[i].Branches != nil {
 			gitParams.Branches = result[i].Branches
 		}
+		result[i].JfrogReleasesRepo = releasesRepo
 		result[i].Git = *gitParams
 		result[i].Server = *server
 		result[i].OutputWriter = GetCompatibleOutputWriter(result[i].GitProvider)
@@ -427,7 +430,7 @@ func getBoolEnv(envKey string, defaultValue bool) (bool, error) {
 }
 
 // In case frogbot-config.yml does not exist, newConfigAggregatorFromEnv generates a FrogbotConfigAggregator with the environment variables values.
-func newConfigAggregatorFromEnv(gitParams *Git, server *coreconfig.ServerDetails) (FrogbotConfigAggregator, error) {
+func newConfigAggregatorFromEnv(gitParams *Git, server *coreconfig.ServerDetails, releasesRepo string) (FrogbotConfigAggregator, error) {
 	// The repo name must be set as a part of the envs.
 	if gitParams.RepoName == "" {
 		return nil, &ErrMissingEnv{GitRepoEnv}
@@ -441,6 +444,7 @@ func newConfigAggregatorFromEnv(gitParams *Git, server *coreconfig.ServerDetails
 	if err := extractRepoParamsFromEnv(&repo); err != nil {
 		return nil, err
 	}
+	repo.JfrogReleasesRepo = releasesRepo
 	repo.Projects = append(repo.Projects, project)
 	repo.OutputWriter = GetCompatibleOutputWriter(gitParams.GitProvider)
 	return FrogbotConfigAggregator{repo}, nil

commands/utils/params_test.go

@@ -146,7 +146,7 @@ func TestExtractAndAssertRepoParams(t *testing.T) {
 	assert.NoError(t, err)
 	configFileContent, err := ReadConfigFromFileSystem(configParamsTestFile)
 	assert.NoError(t, err)
-	configAggregator, err := NewConfigAggregatorFromFile(configFileContent, gitParams, server)
+	configAggregator, err := NewConfigAggregatorFromFile(configFileContent, gitParams, server, "")
 	assert.NoError(t, err)
 	for _, repo := range configAggregator {
 		for projectI, project := range repo.Projects {
@@ -172,7 +172,7 @@ func testExtractAndAssertProjectParams(t *testing.T, project Project) {
 func extractAndAssertParamsFromEnv(t *testing.T, platformUrl, basicAuth bool) {
 	server, gitParams, err := extractEnvParams()
 	assert.NoError(t, err)
-	configFile, err := newConfigAggregatorFromEnv(gitParams, server)
+	configFile, err := newConfigAggregatorFromEnv(gitParams, server, "")
 	assert.NoError(t, err)
 	err = SanitizeEnv()
 	assert.NoError(t, err)
@@ -269,10 +269,11 @@ func TestGenerateConfigAggregatorFromEnv(t *testing.T) {
 		User:           "admin",
 		Password:       "password",
 	}
-	configAggregator, err := newConfigAggregatorFromEnv(&gitParams, &server)
+	configAggregator, err := newConfigAggregatorFromEnv(&gitParams, &server, "releases-remote")
 	assert.NoError(t, err)
 	repo := configAggregator[0]
 	assert.Equal(t, "repoName", repo.RepoName)
+	assert.Equal(t, "releases-remote", repo.JfrogReleasesRepo)
 	assert.ElementsMatch(t, repo.Watches, []string{"watch-1", "watch-2", "watch-3"})
 	assert.Equal(t, false, *repo.FailOnSecurityIssues)
 	assert.Equal(t, gitParams.RepoOwner, repo.RepoOwner)

commands/utils/utils.go

@@ -51,6 +51,7 @@ type ScanDetails struct {
 	Client                   vcsclient.VcsClient
 	FailOnInstallationErrors bool
 	Branch                   string
+	ReleasesRepo             string
 }
 
 // The OutputWriter interface allows Frogbot output to be written in an appropriate way for each git provider.

go.mod

@@ -5,25 +5,25 @@ go 1.19
 require (
 	github.com/go-git/go-git/v5 v5.6.1
 	github.com/golang/mock v1.6.0
-	github.com/jfrog/build-info-go v1.9.0
+	github.com/jfrog/build-info-go v1.9.1
 	github.com/jfrog/froggit-go v1.7.0
 	github.com/jfrog/gofrog v1.2.5
-	github.com/jfrog/jfrog-cli-core/v2 v2.31.0
-	github.com/jfrog/jfrog-client-go v1.28.0
+	github.com/jfrog/jfrog-cli-core/v2 v2.31.1
+	github.com/jfrog/jfrog-client-go v1.28.1
 	github.com/mholt/archiver/v3 v3.5.1
 	github.com/pkg/errors v0.9.1
 	github.com/stretchr/testify v1.8.2
-	github.com/urfave/cli/v2 v2.25.0
+	github.com/urfave/cli/v2 v2.25.1
 	github.com/xeipuuv/gojsonschema v1.2.0
 	gopkg.in/yaml.v2 v2.4.0
 	gopkg.in/yaml.v3 v3.0.1
 )
 
 require (
 	github.com/BurntSushi/toml v1.2.1 // indirect
-	github.com/CycloneDX/cyclonedx-go v0.7.0 // indirect
+	github.com/CycloneDX/cyclonedx-go v0.7.1 // indirect
 	github.com/Microsoft/go-winio v0.5.2 // indirect
-	github.com/ProtonMail/go-crypto v0.0.0-20230217124315-7d5c6f04bbb8 // indirect
+	github.com/ProtonMail/go-crypto v0.0.0-20230331115716-d34776aa93ec // indirect
 	github.com/VividCortex/ewma v1.2.0 // indirect
 	github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d // indirect
 	github.com/acomagu/bufpipe v1.0.4 // indirect
@@ -58,7 +58,7 @@ require (
 	github.com/jedib0t/go-pretty/v6 v6.4.6 // indirect
 	github.com/kevinburke/ssh_config v1.2.0 // indirect
 	github.com/klauspost/compress v1.11.4 // indirect
-	github.com/klauspost/cpuid/v2 v2.0.6 // indirect
+	github.com/klauspost/cpuid/v2 v2.2.3 // indirect
 	github.com/klauspost/pgzip v1.2.5 // indirect
 	github.com/ktrysmt/go-bitbucket v0.9.32 // indirect
 	github.com/magiconair/properties v1.8.7 // indirect
@@ -68,7 +68,7 @@ require (
 	github.com/mattn/go-runewidth v0.0.13 // indirect
 	github.com/mattn/go-tty v0.0.3 // indirect
 	github.com/microsoft/azure-devops-go-api/azuredevops v1.0.0-b5 // indirect
-	github.com/minio/sha256-simd v1.0.1-0.20210617151322-99e45fae3395 // indirect
+	github.com/minio/sha256-simd v1.0.1-0.20230222114820-6096f891a77b // indirect
 	github.com/mitchellh/mapstructure v1.5.0 // indirect
 	github.com/nwaples/rardecode v1.1.0 // indirect
 	github.com/owenrumney/go-sarif/v2 v2.1.3 // indirect
@@ -110,3 +110,9 @@ require (
 	gopkg.in/ini.v1 v1.67.0 // indirect
 	gopkg.in/warnings.v0 v0.1.2 // indirect
 )
+
+//replace github.com/jfrog/jfrog-client-go => github.com/jfrog/jfrog-client-go v1.13.2-0.20230404093739-610cb3af8862
+
+//replace github.com/jfrog/jfrog-cli-core/v2 => github.com/jfrog/jfrog-cli-core/v2 v2.15.3-0.20230404095831-ce76a2622e41
+
+//replace github.com/jfrog/build-info-go => github.com/jfrog/build-info-go v1.8.9-0.20230403064815-ea83b399ac8e