Add Xray audit for pip and go (#229)

* Add support for commands with build and project flags combined, and also api to artifactory Access/projects RestApi.

Signed-off-by: sverdlov93 <[email protected]>

* artDetails --> serviceDetails CR change

Signed-off-by: sverdlov93 <[email protected]>

* change access/api to access

Signed-off-by: sverdlov93 <[email protected]>

* Add access access token to fix projects test

Signed-off-by: sverdlov93 <[email protected]>

* Fix Access Url test

Signed-off-by: sverdlov93 <[email protected]>

* Fix Access Url test

Signed-off-by: sverdlov93 <[email protected]>

* Fix Access Url test

Signed-off-by: sverdlov93 <[email protected]>

* Add Xray Go Audit Scan and Go Cmd utils and unite audit code

Signed-off-by: sverdlov93 <[email protected]>

* Add Xray Go Audit Scan cli command

Signed-off-by: sverdlov93 <[email protected]>

* Fix python Xray packageType: "pip" > "pypi"

Signed-off-by: sverdlov93 <[email protected]>

* Add xr audit-pip feature and audit pip and go tests

Signed-off-by: sverdlov93 <[email protected]>

* Update config_test.go

* Update utils.go

* Update configtoken_test.go

* Remove unnecessary code from pip utils

Signed-off-by: sverdlov93 <[email protected]>

* Update go.mod with replace for gocmd

Signed-off-by: sverdlov93 <[email protected]>

* Some Changes after CR

Signed-off-by: sverdlov93 <[email protected]>

* Some Changes after CR

Signed-off-by: sverdlov93 <[email protected]>

* Improve error log for non existing build info dir

Signed-off-by: sverdlov93 <[email protected]>

* remove pipdeptree script from gitignore

Signed-off-by: sverdlov93 <[email protected]>

* update go.mod and go.sum

Signed-off-by: sverdlov93 <[email protected]>

* some changes for CR2

Signed-off-by: sverdlov93 <[email protected]>

.gitignore

@@ -9,9 +9,6 @@ testdata/go/tmp
 tmp
 out
 
-# Pipdeptree
-artifactory/utils/pip/dependencies/deptreescript.go
-
 # Vim
 *~
 *.swp

artifactory/commands/golang/go.go

@@ -8,9 +8,9 @@ import (
 	executors "github.com/jfrog/gocmd/executers/utils"
 	"github.com/jfrog/gocmd/params"
 	"github.com/jfrog/jfrog-cli-core/v2/artifactory/utils"
-	"github.com/jfrog/jfrog-cli-core/v2/artifactory/utils/golang"
 	"github.com/jfrog/jfrog-cli-core/v2/artifactory/utils/golang/project"
 	"github.com/jfrog/jfrog-cli-core/v2/utils/config"
+	goutils "github.com/jfrog/jfrog-cli-core/v2/utils/golang"
 	"github.com/jfrog/jfrog-client-go/auth"
 	"github.com/jfrog/jfrog-client-go/utils/errorutils"
 	"github.com/jfrog/jfrog-client-go/utils/io/fileutils"
@@ -110,7 +110,7 @@ func (gc *GoCommand) Run() error {
 }
 
 func (gc *GoCommand) run() error {
-	err := golang.LogGoVersion()
+	err := goutils.LogGoVersion()
 	if err != nil {
 		return err
 	}

artifactory/commands/golang/go_test.go

@@ -1,7 +1,7 @@
 package golang
 
 import (
-	"github.com/jfrog/gocmd/cmd"
+	goutils "github.com/jfrog/jfrog-cli-core/v2/utils/golang"
 	"github.com/stretchr/testify/assert"
 	"os"
 	"path/filepath"
@@ -29,7 +29,7 @@ func TestBuildPackageVersionRequest(t *testing.T) {
 }
 
 func TestGetPackageFilesPath(t *testing.T) {
-	packageCachePath, err := cmd.GetGoModCachePath()
+	packageCachePath, err := goutils.GetGoModCachePath()
 	assert.NoError(t, err)
 	packageName := "github.com/golang/mock/mockgen"
 	version := "v1.4.1"

artifactory/commands/golang/gopublish.go

@@ -4,8 +4,8 @@ import (
 	"errors"
 	commandutils "github.com/jfrog/jfrog-cli-core/v2/artifactory/commands/utils"
 	"github.com/jfrog/jfrog-cli-core/v2/artifactory/utils"
-	"github.com/jfrog/jfrog-cli-core/v2/artifactory/utils/golang"
 	"github.com/jfrog/jfrog-cli-core/v2/artifactory/utils/golang/project"
+	goutils "github.com/jfrog/jfrog-cli-core/v2/utils/golang"
 	"github.com/jfrog/jfrog-client-go/utils/errorutils"
 	"github.com/jfrog/jfrog-client-go/utils/version"
 	"os/exec"
@@ -46,7 +46,7 @@ func (gpc *GoPublishCommand) Run() error {
 		return err
 	}
 
-	err = golang.LogGoVersion()
+	err = goutils.LogGoVersion()
 	if err != nil {
 		return err
 	}

artifactory/commands/npm/installorci.go

@@ -7,6 +7,7 @@ import (
 	"github.com/jfrog/jfrog-client-go/utils/version"
 	"io/ioutil"
 	"os"
+	"os/exec"
 	"path/filepath"
 	"strconv"
 	"strings"
@@ -168,11 +169,15 @@ func (nca *NpmCommandArgs) run() error {
 
 func (nca *NpmCommandArgs) preparePrerequisites(repo string) error {
 	log.Debug("Preparing prerequisites.")
-	path, err := npmutils.FindNpmExecutable()
+	npmExecPath, err := exec.LookPath("npm")
 	if err != nil {
-		return err
+		return errorutils.CheckError(err)
+	}
+
+	if npmExecPath == "" {
+		return errorutils.CheckError(errors.New("could not find the 'npm' executable in the system PATH"))
 	}
-	nca.executablePath = path
+	nca.executablePath = npmExecPath
 
 	if err = nca.validateNpmVersion(); err != nil {
 		return err

artifactory/commands/pip/install.go

@@ -4,6 +4,7 @@ import (
 	"errors"
 	"fmt"
 	"os"
+	"os/exec"
 	"path/filepath"
 	"strings"
 
@@ -79,7 +80,10 @@ func (pic *PipInstallCommand) collectBuildInfo(pythonExecutablePath string, depe
 	}
 
 	promptMissingDependencies(missingDeps)
-	dependencies.UpdateDependenciesCache(allDependencies)
+	err = dependencies.UpdateDependenciesCache(allDependencies)
+	if err != nil {
+		return err
+	}
 	pic.saveBuildInfo(allDependencies)
 	return nil
 }
@@ -135,11 +139,13 @@ func (pic *PipInstallCommand) determineModuleName(pythonExecutablePath string) e
 func (pic *PipInstallCommand) prepare() (pythonExecutablePath string, err error) {
 	log.Debug("Preparing prerequisites.")
 
-	pythonExecutablePath, err = piputils.GetExecutablePath("python")
+	pythonExecutablePath, err = exec.LookPath("python")
 	if err != nil {
 		return
 	}
-
+	if pythonExecutablePath == "" {
+		return "", errorutils.CheckError(errors.New("Could not find the 'python' executable in the system PATH"))
+	}
 	pic.args, pic.buildConfiguration, err = utils.ExtractBuildDetailsFromArgs(pic.args)
 	if err != nil {
 		return

artifactory/utils/buildutils.go

@@ -172,7 +172,10 @@ func GetGeneratedBuildsInfo(buildName, buildNumber, projectKey string) ([]*build
 			return nil, err
 		}
 		buildInfo := new(buildinfo.BuildInfo)
-		json.Unmarshal(content, &buildInfo)
+		err = json.Unmarshal(content, &buildInfo)
+		if errorutils.CheckError(err) != nil {
+			return nil, err
+		}
 		generatedBuildsInfo = append(generatedBuildsInfo, buildInfo)
 	}
 	return generatedBuildsInfo, nil
@@ -204,7 +207,10 @@ func ReadPartialBuildInfoFiles(buildName, buildNumber, projectKey string) (build
 			return nil, err
 		}
 		partial := new(buildinfo.Partial)
-		json.Unmarshal(content, &partial)
+		err = json.Unmarshal(content, &partial)
+		if errorutils.CheckError(err) != nil {
+			return nil, err
+		}
 		partials = append(partials, partial)
 	}
 
@@ -217,12 +223,29 @@ func ReadBuildInfoGeneralDetails(buildName, buildNumber, projectKey string) (*bu
 		return nil, err
 	}
 	generalDetailsFilePath := filepath.Join(partialsBuildDir, BuildInfoDetails)
+	fileExists, err := fileutils.IsFileExists(generalDetailsFilePath, false)
+	if err != nil {
+		return nil, err
+	}
+	if fileExists == false {
+		var buildString string
+		if projectKey != "" {
+			buildString = fmt.Sprintf("build-name: <%s>, build-number: <%s> and project: <%s>", buildName, buildNumber, projectKey)
+		} else {
+			buildString = fmt.Sprintf("build-name: <%s> and build-number: <%s>", buildName, buildNumber)
+		}
+		return nil, errors.New("Failed to construct the build-info to be published. " +
+			"This may be because there were no previous commands, which collected build-info for " + buildString)
+	}
 	content, err := fileutils.ReadFile(generalDetailsFilePath)
 	if err != nil {
 		return nil, err
 	}
 	details := new(buildinfo.General)
-	json.Unmarshal(content, &details)
+	err = json.Unmarshal(content, &details)
+	if errorutils.CheckError(err) != nil {
+		return nil, err
+	}
 	return details, nil
 }
 

artifactory/utils/golang/project/project.go

@@ -9,14 +9,12 @@ import (
 	"io/ioutil"
 	"os"
 	"path/filepath"
-	"regexp"
 	"strings"
 	"time"
 
-	"github.com/jfrog/gocmd/cmd"
-	gocmd "github.com/jfrog/gocmd/cmd"
 	"github.com/jfrog/gocmd/executers"
 	"github.com/jfrog/jfrog-cli-core/v2/artifactory/utils"
+	goutils "github.com/jfrog/jfrog-cli-core/v2/utils/golang"
 	"github.com/jfrog/jfrog-client-go/artifactory"
 	"github.com/jfrog/jfrog-client-go/artifactory/buildinfo"
 	_go "github.com/jfrog/jfrog-client-go/artifactory/services/go"
@@ -80,11 +78,11 @@ func (project *goProject) LoadDependencies() error {
 }
 
 func (project *goProject) loadDependencies() ([]executers.Package, error) {
-	cachePath, err := gocmd.GetCachePath()
+	cachePath, err := goutils.GetCachePath()
 	if err != nil {
 		return nil, err
 	}
-	modulesMap, err := cmd.GetDependenciesList(project.projectPath)
+	modulesMap, err := goutils.GetDependenciesList(project.projectPath)
 	if err != nil {
 		return nil, err
 	}
@@ -241,7 +239,7 @@ func (project *goProject) getId() string {
 func (project *goProject) readModFile() error {
 	var err error
 	if project.projectPath == "" {
-		project.projectPath, err = cmd.GetProjectRoot()
+		project.projectPath, err = goutils.GetProjectRoot()
 		if err != nil {
 			return errorutils.CheckError(err)
 		}
@@ -263,7 +261,7 @@ func (project *goProject) readModFile() error {
 	}
 
 	// Read module name
-	project.moduleName, err = parseModuleName(string(content))
+	project.moduleName, err = goutils.GetModuleName(project.projectPath)
 	if err != nil {
 		return err
 	}
@@ -340,23 +338,6 @@ func (project *goProject) addInfoFileToBuildInfo(infoFilePath string) error {
 	return nil
 }
 
-// Parse module name from go.mod content.
-func parseModuleName(modContent string) (string, error) {
-	r, err := regexp.Compile(`module "?([\w\.@:%_\+-.~#?&]+/?.+\w)`)
-	if err != nil {
-		return "", errorutils.CheckError(err)
-	}
-	lines := strings.Split(modContent, "\n")
-	for _, v := range lines {
-		matches := r.FindStringSubmatch(v)
-		if len(matches) == 2 {
-			return matches[1], nil
-		}
-	}
-
-	return "", errorutils.CheckError(errors.New("Module name missing in go.mod file"))
-}
-
 type goInfo struct {
 	Version string `json:"Version"`
 	Time    string `json:"Time"`

artifactory/utils/pip/installer.go

@@ -1,8 +1,10 @@
 package pip
 
 import (
+	"errors"
 	"fmt"
 	"net/url"
+	"os/exec"
 	"strings"
 
 	gofrogcmd "github.com/jfrog/gofrog/io"
@@ -40,16 +42,14 @@ func (pi *PipInstaller) Install() error {
 func (pi *PipInstaller) prepare() (pipExecutablePath, pipIndexUrl string, err error) {
 	log.Debug("Preparing prerequisites.")
 
-	pipExecutablePath, err = GetExecutablePath("pip")
+	pipExecutablePath, err = exec.LookPath("pip")
 	if err != nil {
 		return
 	}
-
-	pipIndexUrl, err = getArtifactoryUrlWithCredentials(pi.ServerDetails, pi.Repository)
-	if err != nil {
-		return
+	if pipExecutablePath == "" {
+		return "", "", errorutils.CheckError(errors.New("Could not find the 'pip' executable in the system PATH"))
 	}
-
+	pipIndexUrl, err = getArtifactoryUrlWithCredentials(pi.ServerDetails, pi.Repository)
 	return
 }