diff --git a/.github/workflows/frogbot-scan-and-fix.yml b/.github/workflows/frogbot-scan-and-fix.yml deleted file mode 100644 index aa34c12cc..000000000 --- a/.github/workflows/frogbot-scan-and-fix.yml +++ /dev/null @@ -1,40 +0,0 @@ -name: "Frogbot Scan and Fix" -on: - schedule: - # The repository will be scanned once a day at 00:00 GMT. - - cron: "0 0 * * *" -permissions: - contents: write - pull-requests: write - security-events: write -jobs: - create-fix-pull-requests: - runs-on: ubuntu-latest - strategy: - matrix: - # The repository scanning will be triggered periodically on the following branches. - branch: [ "dev" ] - steps: - - uses: actions/checkout@v3 - with: - ref: ${{ matrix.branch }} - - # Install prerequisites - - name: Setup Go - uses: actions/setup-go@v3 - with: - go-version: 1.20.x - - - uses: jfrog/frogbot@v2 - env: - # [Mandatory] - # JFrog platform URL - JF_URL: ${{ secrets.FROGBOT_URL }} - - # [Mandatory if JF_USER and JF_PASSWORD are not provided] - # JFrog access token with 'read' permissions on Xray service - JF_ACCESS_TOKEN: ${{ secrets.FROGBOT_ACCESS_TOKEN }} - - # [Mandatory] - # The GitHub token automatically generated for the job - JF_GIT_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/frogbot-scan-pr.yml b/.github/workflows/frogbot-scan-pr.yml deleted file mode 100644 index d57c83e98..000000000 --- a/.github/workflows/frogbot-scan-pr.yml +++ /dev/null @@ -1,37 +0,0 @@ -name: "Frogbot Scan Pull Request" -on: - pull_request_target: - types: [ opened, synchronize ] -permissions: - pull-requests: write - contents: read -jobs: - scan-pull-request: - runs-on: ubuntu-latest - # A pull request needs to be approved, before Frogbot scans it. Any GitHub user who is associated with the - # "frogbot" GitHub environment can approve the pull request to be scanned. - environment: frogbot - steps: - - uses: actions/checkout@v3 - with: - ref: ${{ github.event.pull_request.head.sha }} - - # Install prerequisites - - name: Setup Go - uses: actions/setup-go@v3 - with: - go-version: 1.20.x - - - uses: jfrog/frogbot@v2 - env: - # [Mandatory] - # JFrog platform URL (This functionality requires version 3.29.0 or above of Xray) - JF_URL: ${{ secrets.FROGBOT_URL }} - - # [Mandatory if JF_USER and JF_PASSWORD are not provided] - # JFrog access token with 'read' permissions on Xray service - JF_ACCESS_TOKEN: ${{ secrets.FROGBOT_ACCESS_TOKEN }} - - # [Mandatory] - # The GitHub token automatically generated for the job - JF_GIT_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/frogbot-scan-pull-request.yml b/.github/workflows/frogbot-scan-pull-request.yml new file mode 100644 index 000000000..998c8c91f --- /dev/null +++ b/.github/workflows/frogbot-scan-pull-request.yml @@ -0,0 +1,119 @@ +name: "Frogbot Scan Pull Request" +on: + pull_request_target: + types: [ opened, synchronize ] +permissions: + pull-requests: write + contents: read +jobs: + scan-pull-request: + runs-on: ubuntu-latest + # A pull request needs to be approved before Frogbot scans it. Any GitHub user who is associated with the + # "frogbot" GitHub environment can approve the pull request to be scanned. + environment: frogbot + steps: + - uses: jfrog/frogbot@v2 + env: + JFROG_CLI_LOG_LEVEL: "DEBUG" + # [Mandatory] + # JFrog platform URL (This functionality requires version 3.29.0 or above of Xray) + JF_URL: ${{ secrets.FROGBOT_URL }} + + # [Mandatory if JF_USER and JF_PASSWORD are not provided] + # JFrog access token with 'read' permissions on Xray service + JF_ACCESS_TOKEN: ${{ secrets.FROGBOT_ACCESS_TOKEN }} + + # [Mandatory] + # The GitHub token is automatically generated for the job + JF_GIT_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + # [Optional, default: https://api.github.com] + # API endpoint to GitHub + # JF_GIT_API_ENDPOINT: https://github.example.com + + # [Optional] + # By default, the Frogbot workflows download the Frogbot executable as well as other tools + # needed from https://releases.jfrog.io + # If the machine that runs Frogbot has no access to the internet, follow these steps to allow the + # executable to be downloaded from an Artifactory instance, which the machine has access to: + # + # 1. Login to the Artifactory UI, with a user who has admin credentials. + # 2. Create a Remote Repository with the following properties set. + # Under the 'Basic' tab: + # Package Type: Generic + # URL: https://releases.jfrog.io + # Under the 'Advanced' tab: + # Uncheck the 'Store Artifacts Locally' option + # 3. Set the value of the 'JF_RELEASES_REPO' variable with the Repository Key you created. + # JF_RELEASES_REPO: "" + + # [Optional] + # Configure the SMTP server to enable Frogbot to send emails with detected secrets in pull request scans. + # SMTP server URL including should the relevant port: (Example: smtp.server.com:8080) + JF_SMTP_SERVER: ${{ secrets.JF_SMTP_SERVER }} + + # [Mandatory if JF_SMTP_SERVER is set] + # The username required for authenticating with the SMTP server. + JF_SMTP_USER: ${{ secrets.JF_SMTP_USER }} + + # [Mandatory if JF_SMTP_SERVER is set] + # The password associated with the username required for authentication with the SMTP server. + JF_SMTP_PASSWORD: ${{ secrets.JF_SMTP_PASSWORD }} + + # [Optional] + # List of comma separated email addresses to receive email notifications about secrets + # detected during pull request scanning. The notification is also sent to the email set + # in the committer git profile regardless of whether this variable is set or not. + JF_EMAIL_RECEIVERS: "eco-system@jfrog.com" + + ########################################################################## + ## If your project uses a 'frogbot-config.yml' file, you can define ## + ## the following variables inside the file, instead of here. ## + ########################################################################## + + # [Mandatory if the two conditions below are met] + # 1. The project uses yarn 2, NuGet or .NET Core to download its dependencies + # 2. The `installCommand` variable isn't set in your frogbot-config.yml file. + # + # The command that installs the project dependencies (e.g "nuget restore") + # JF_INSTALL_DEPS_CMD: "" + + # [Optional, default: "."] + # Relative path to the root of the project in the Git repository + # JF_WORKING_DIR: path/to/project/dir + + # [Optional] + # Xray Watches. Learn more about them here: https://www.jfrog.com/confluence/display/JFROG/Configuring+Xray+Watches + # JF_WATCHES: ,... + + # [Optional] + # JFrog project. Learn more about it here: https://www.jfrog.com/confluence/display/JFROG/Projects + # JF_PROJECT: + + # [Optional, default: "FALSE"] + # Displays all existing vulnerabilities, including the ones that were added by the pull request. + # JF_INCLUDE_ALL_VULNERABILITIES: "TRUE" + + # [Optional, default: "TRUE"] + # Fails the Frogbot task if any security issue is found. + # JF_FAIL: "FALSE" + + # [Optional] + # Frogbot will download the project dependencies if they're not cached locally. To download the + # dependencies from a virtual repository in Artifactory, set the name of the repository. There's no + # need to set this value, if it is set in the frogbot-config.yml file. + # JF_DEPS_REPO: "" + + # [Optional, Default: "FALSE"] + # If TRUE, Frogbot creates a single pull request with all the fixes. + # If false, Frogbot creates a separate pull request for each fix. + # JF_GIT_AGGREGATE_FIXES: "FALSE" + + # [Optional, Default: "FALSE"] + # Handle vulnerabilities with fix versions only + # JF_FIXABLE_ONLY: "TRUE" + + # [Optional] + # Set the minimum severity for vulnerabilities that should be fixed and commented on in pull requests + # The following values are accepted: Low, Medium, High or Critical + # JF_MIN_SEVERITY: "" \ No newline at end of file diff --git a/.github/workflows/frogbot-scan-repository.yml b/.github/workflows/frogbot-scan-repository.yml new file mode 100644 index 000000000..01b568f67 --- /dev/null +++ b/.github/workflows/frogbot-scan-repository.yml @@ -0,0 +1,125 @@ +name: "Frogbot Scan Repository" +on: + workflow_dispatch: + schedule: + # The repository will be scanned once a day at 00:00 GMT. + - cron: "0 0 * * *" +permissions: + contents: write + pull-requests: write + security-events: write +jobs: + scan-repository: + runs-on: ubuntu-latest + strategy: + matrix: + # The repository scanning will be triggered periodically on the following branches. + branch: [ "dev" ] + steps: + - uses: jfrog/frogbot@v2 + env: + JFROG_CLI_LOG_LEVEL: "DEBUG" + # [Mandatory] + # JFrog platform URL (This functionality requires version 3.29.0 or above of Xray) + JF_URL: ${{ secrets.FROGBOT_URL }} + + # [Mandatory if JF_USER and JF_PASSWORD are not provided] + # JFrog access token with 'read' permissions on Xray service + JF_ACCESS_TOKEN: ${{ secrets.FROGBOT_ACCESS_TOKEN }} + + # [Mandatory if JF_ACCESS_TOKEN is not provided] + # JFrog username with 'read' permissions for Xray. Must be provided with JF_PASSWORD + # JF_USER: ${{ secrets.JF_USER }} + + # [Mandatory if JF_ACCESS_TOKEN is not provided] + # JFrog password. Must be provided with JF_USER + # JF_PASSWORD: ${{ secrets.JF_PASSWORD }} + + # [Mandatory] + # The GitHub token is automatically generated for the job + JF_GIT_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + # [Mandatory] + # The name of the branch on which Frogbot will perform the scan + JF_GIT_BASE_BRANCH: ${{ matrix.branch }} + + # [Optional, default: https://api.github.com] + # API endpoint to GitHub + # JF_GIT_API_ENDPOINT: https://github.example.com + + # [Optional] + # By default, the Frogbot workflows download the Frogbot executable as well as other tools + # needed from https://releases.jfrog.io + # If the machine that runs Frogbot has no access to the internet, follow these steps to allow the + # executable to be downloaded from an Artifactory instance, which the machine has access to: + # + # 1. Login to the Artifactory UI, with a user who has admin credentials. + # 2. Create a Remote Repository with the following properties set. + # Under the 'Basic' tab: + # Package Type: Generic + # URL: https://releases.jfrog.io + # Under the 'Advanced' tab: + # Uncheck the 'Store Artifacts Locally' option + # 3. Set the value of the 'JF_RELEASES_REPO' variable with the Repository Key you created. + # JF_RELEASES_REPO: "" + + ########################################################################## + ## If your project uses a 'frogbot-config.yml' file, you can define ## + ## the following variables inside the file, instead of here. ## + ########################################################################## + + # [Optional, default: "."] + # Relative path to the root of the project in the Git repository + # JF_WORKING_DIR: path/to/project/dir + + # [Optional] + # Xray Watches. Learn more about them here: https://www.jfrog.com/confluence/display/JFROG/Configuring+Xray+Watches + # JF_WATCHES: ,... + + # [Optional] + # JFrog project. Learn more about it here: https://www.jfrog.com/confluence/display/JFROG/Projects + # JF_PROJECT: + + # [Optional, default: "TRUE"] + # Fails the Frogbot task if any security issue is found. + # JF_FAIL: "FALSE" + + # [Optional] + # Frogbot will download the project dependencies, if they're not cached locally. To download the + # dependencies from a virtual repository in Artifactory, set the name of the repository. There's no + # need to set this value, if it is set in the frogbot-config.yml file. + # JF_DEPS_REPO: "" + + # [Optional] + # Template for the branch name generated by Frogbot when creating pull requests with fixes. + # The template must include ${BRANCH_NAME_HASH}, to ensure that the generated branch name is unique. + # The template can optionally include the ${IMPACTED_PACKAGE} and ${FIX_VERSION} variables. + # JF_BRANCH_NAME_TEMPLATE: "frogbot-${IMPACTED_PACKAGE}-${BRANCH_NAME_HASH}" + + # [Optional] + # Template for the commit message generated by Frogbot when creating pull requests with fixes + # The template can optionally include the ${IMPACTED_PACKAGE} and ${FIX_VERSION} variables. + # JF_COMMIT_MESSAGE_TEMPLATE: "Upgrade ${IMPACTED_PACKAGE} to ${FIX_VERSION}" + + # [Optional] + # Template for the pull request title generated by Frogbot when creating pull requests with fixes. + # The template can optionally include the ${IMPACTED_PACKAGE} and ${FIX_VERSION} variables. + # JF_PULL_REQUEST_TITLE_TEMPLATE: "[🐸 Frogbot] Upgrade ${IMPACTED_PACKAGE} to ${FIX_VERSION}" + + # [Optional, Default: "FALSE"] + # If TRUE, Frogbot creates a single pull request with all the fixes. + # If FALSE, Frogbot creates a separate pull request for each fix. + # JF_GIT_AGGREGATE_FIXES: "FALSE" + + # [Optional, Default: "FALSE"] + # Handle vulnerabilities with fix versions only + # JF_FIXABLE_ONLY: "TRUE" + + # [Optional] + # Set the minimum severity for vulnerabilities that should be fixed and commented on in pull requests + # The following values are accepted: Low, Medium, High or Critical + # JF_MIN_SEVERITY: "" + + # [Optional, Default: eco-system+frogbot@jfrog.com] + # Set the email of the commit author + # JF_GIT_EMAIL_AUTHOR: "" \ No newline at end of file diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index e06472f52..a729bbc44 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -12,7 +12,7 @@ jobs: strategy: fail-fast: false matrix: - os: [ ubuntu, windows, macOS ] + os: [ ubuntu, windows, macos ] env: GOPROXY: direct GRADLE_OPTS: -Dorg.gradle.daemon=false diff --git a/artifactory/commands/golang/go.go b/artifactory/commands/golang/go.go index 22dda146f..6395c5b0c 100644 --- a/artifactory/commands/golang/go.go +++ b/artifactory/commands/golang/go.go @@ -4,6 +4,7 @@ import ( "encoding/json" "errors" "fmt" + "github.com/jfrog/build-info-go/build" biutils "github.com/jfrog/build-info-go/utils" "github.com/jfrog/jfrog-cli-core/v2/artifactory/utils" "github.com/jfrog/jfrog-cli-core/v2/utils/config" @@ -15,7 +16,9 @@ import ( "github.com/jfrog/jfrog-client-go/utils/errorutils" "github.com/jfrog/jfrog-client-go/utils/io/fileutils" "github.com/jfrog/jfrog-client-go/utils/log" + "io/fs" "net/http" + "os" "path" "path/filepath" "strings" @@ -131,39 +134,37 @@ func (gc *GoCommand) extractNoFallbackFromArgs() (cleanArgs []string, err error) return } -func (gc *GoCommand) run() error { - err := goutils.LogGoVersion() +func (gc *GoCommand) run() (err error) { + err = goutils.LogGoVersion() if err != nil { - return err + return } goBuildInfo, err := utils.PrepareBuildPrerequisites(gc.buildConfiguration) if err != nil { - return err + return } defer func() { if goBuildInfo != nil && err != nil { - e := goBuildInfo.Clean() - if e != nil { - err = errors.New(err.Error() + "\n" + e.Error()) - } + err = errors.Join(err, goBuildInfo.Clean()) } }() resolverDetails, err := gc.resolverParams.ServerDetails() if err != nil { - return err + return } repoUrl, err := goutils.GetArtifactoryRemoteRepoUrl(resolverDetails, gc.resolverParams.TargetRepo()) if err != nil { - return err + return } // If noFallback=false, missing packages will be fetched directly from VCS if !gc.noFallback { repoUrl += "|direct" } err = biutils.RunGo(gc.goArg, repoUrl) - if err != nil { - return coreutils.ConvertExitCodeError(errorutils.CheckError(err)) + if errorutils.CheckError(err) != nil { + err = coreutils.ConvertExitCodeError(err) + return } if goBuildInfo != nil { @@ -172,39 +173,39 @@ func (gc *GoCommand) run() error { if isGoGetCommand := len(gc.goArg) > 0 && gc.goArg[0] == "get"; isGoGetCommand { if len(gc.goArg) < 2 { // Package name was not supplied. Invalid go get commend - return errorutils.CheckErrorf("Invalid get command. Package name is missing") + err = errorutils.CheckErrorf("Invalid get command. Package name is missing") + return } tempDirPath, err = fileutils.CreateTempDir() if err != nil { - return err + return } // Cleanup the temp working directory at the end. defer func() { - e := fileutils.RemoveTempDir(tempDirPath) - if err == nil { - err = e - } + err = errors.Join(err, fileutils.RemoveTempDir(tempDirPath)) }() - serverDetails, err := resolverDetails.CreateArtAuthConfig() + var serverDetails auth.ServiceDetails + serverDetails, err = resolverDetails.CreateArtAuthConfig() if err != nil { - return err + return } err = copyGoPackageFiles(tempDirPath, gc.goArg[1], gc.resolverParams.TargetRepo(), serverDetails) if err != nil { - return err + return } } - goModule, err := goBuildInfo.AddGoModule(tempDirPath) - if err != nil { - return errorutils.CheckError(err) + var goModule *build.GoModule + goModule, err = goBuildInfo.AddGoModule(tempDirPath) + if errorutils.CheckError(err) != nil { + return } if gc.buildConfiguration.GetModule() != "" { goModule.SetName(gc.buildConfiguration.GetModule()) } - return errorutils.CheckError(goModule.CalcDependencies()) + err = errorutils.CheckError(goModule.CalcDependencies()) } - return err + return } // copyGoPackageFiles copies the package files from the go mod cache directory to the given destPath. @@ -215,11 +216,21 @@ func copyGoPackageFiles(destPath, packageName, rtTargetRepo string, authArtDetai return err } // Copy the entire content of the relevant Go pkg directory to the requested destination path. - err = fileutils.CopyDir(packageFilesPath, destPath, true, nil) + err = biutils.CopyDir(packageFilesPath, destPath, true, nil) if err != nil { return fmt.Errorf("couldn't find suitable package files: %s", packageFilesPath) } - return nil + // Set permission recursively + return filepath.WalkDir(destPath, func(path string, info fs.DirEntry, err error) error { + if err != nil { + return err + } + err = os.Chmod(path, 0700) + if err != nil { + return err + } + return nil + }) } // getPackageFilePathFromArtifactory returns a string that represents the package files cache path. @@ -249,11 +260,8 @@ func getPackageFilePathFromArtifactory(packageName, rtTargetRepo string, authArt return } } - path, err := getFileSystemPackagePath(packageCachePath, name, version) - if err != nil { - return "", err - } - return path, nil + packageFilesPath, err = getFileSystemPackagePath(packageCachePath, name, version) + return } @@ -315,7 +323,7 @@ func getFileSystemPackagePath(packageCachePath, name, version string) (string, e path, _ = filepath.Split(path) path = strings.TrimSuffix(path, separator) } - return "", errors.New("Could not find package:" + name + " in:" + packageCachePath) + return "", errors.New("Could not find package: " + name + " in: " + packageCachePath) } // buildPackageVersionRequest returns a string representing the version request to Artifactory. diff --git a/artifactory/commands/npm/npmcommand.go b/artifactory/commands/npm/npmcommand.go index ca69e787b..c6b97cff7 100644 --- a/artifactory/commands/npm/npmcommand.go +++ b/artifactory/commands/npm/npmcommand.go @@ -2,6 +2,7 @@ package npm import ( "bufio" + "errors" "fmt" "github.com/jfrog/build-info-go/build" biutils "github.com/jfrog/build-info-go/build/utils" @@ -159,7 +160,7 @@ func (ca *NpmCommand) PreparePrerequisites(repo string) error { } func (ca *NpmCommand) setRestoreNpmrcFunc() error { - restoreNpmrcFunc, err := commandUtils.BackupFile(filepath.Join(ca.workingDirectory, npmrcFileName), filepath.Join(ca.workingDirectory, npmrcBackupFileName)) + restoreNpmrcFunc, err := commandUtils.BackupFile(filepath.Join(ca.workingDirectory, npmrcFileName), npmrcBackupFileName) if err != nil { return err } @@ -169,7 +170,7 @@ func (ca *NpmCommand) setRestoreNpmrcFunc() error { } return restoreNpmrcFunc() } - return err + return nil } func (ca *NpmCommand) setArtifactoryAuth() error { @@ -280,10 +281,7 @@ func (ca *NpmCommand) Run() (err error) { return } defer func() { - e := ca.restoreNpmrcFunc() - if err == nil { - err = e - } + err = errors.Join(err, ca.restoreNpmrcFunc()) }() if err = ca.CreateTempNpmrc(); err != nil { return @@ -293,9 +291,7 @@ func (ca *NpmCommand) Run() (err error) { return } - if err = ca.collectDependencies(); err != nil { - return - } + err = ca.collectDependencies() return } diff --git a/artifactory/commands/transferconfig/utils_test.go b/artifactory/commands/transferconfig/utils_test.go index 4406a951c..4fd79e3b0 100644 --- a/artifactory/commands/transferconfig/utils_test.go +++ b/artifactory/commands/transferconfig/utils_test.go @@ -3,6 +3,7 @@ package transferconfig import ( "archive/zip" "bytes" + biutils "github.com/jfrog/build-info-go/utils" "github.com/jfrog/jfrog-client-go/utils/io/fileutils" "golang.org/x/exp/slices" "io" @@ -43,7 +44,7 @@ func initHandleTypoInAccessBootstrapTest(t *testing.T) (exportDir string, cleanu exportDir, err := fileutils.CreateTempDir() assert.NoError(t, err) testDataPath := filepath.Join("..", "testdata", "artifactory_export") - assert.NoError(t, fileutils.CopyDir(testDataPath, exportDir, true, nil)) + assert.NoError(t, biutils.CopyDir(testDataPath, exportDir, true, nil)) cleanupFunc = func() { assert.NoError(t, fileutils.RemoveTempDir(exportDir), "Couldn't remove temp dir") } diff --git a/artifactory/commands/transferinstall/datatransferinstall.go b/artifactory/commands/transferinstall/datatransferinstall.go index 0c4ac6371..8381966fe 100644 --- a/artifactory/commands/transferinstall/datatransferinstall.go +++ b/artifactory/commands/transferinstall/datatransferinstall.go @@ -2,7 +2,7 @@ package transferinstall import ( "fmt" - downloadutils "github.com/jfrog/build-info-go/utils" + biutils "github.com/jfrog/build-info-go/utils" "github.com/jfrog/gofrog/version" "github.com/jfrog/jfrog-cli-core/v2/artifactory/utils" "github.com/jfrog/jfrog-cli-core/v2/utils/config" @@ -293,7 +293,7 @@ func DownloadFiles(src string, pluginDir string, bundle PluginFiles) (err error) if err = fileutils.CreateDirIfNotExist(dstDirPath); err != nil { return } - if err = downloadutils.DownloadFile(filepath.Join(dstDirPath, fileName), srcURL); err != nil { + if err = biutils.DownloadFile(filepath.Join(dstDirPath, fileName), srcURL); err != nil { err = downloadConnectionErr(src, fileName, err.Error()) return } @@ -311,7 +311,7 @@ func CopyFiles(src string, pluginDir string, bundle PluginFiles) (err error) { if err = fileutils.CreateDirIfNotExist(dstDirPath); err != nil { return } - if err = fileutils.CopyFile(dstDirPath, srcPath); err != nil { + if err = biutils.CopyFile(dstDirPath, srcPath); err != nil { return } } diff --git a/artifactory/commands/utils/npmcmdutils.go b/artifactory/commands/utils/npmcmdutils.go index a9c3dcc7f..883fa88d2 100644 --- a/artifactory/commands/utils/npmcmdutils.go +++ b/artifactory/commands/utils/npmcmdutils.go @@ -1,14 +1,16 @@ package utils import ( + "errors" "fmt" + "io" "net/http" "os" + "path/filepath" "strings" xrutils "github.com/jfrog/jfrog-cli-core/v2/xray/utils" - "github.com/jfrog/jfrog-cli-core/v2/utils/ioutils" "github.com/jfrog/jfrog-client-go/utils/io/fileutils" "github.com/jfrog/jfrog-cli-core/v2/artifactory/utils" @@ -114,28 +116,51 @@ func ExtractNpmOptionsFromArgs(args []string) (detailedSummary, xrayScan bool, s // BackupFile creates a backup of the file in filePath. The backup will be found at backupPath. // The returned restore function can be called to restore the file's state - the file in filePath will be replaced by the backup in backupPath. // If there is no file at filePath, a backup file won't be created, and the restore function will delete the file at filePath. -func BackupFile(filePath, backupPath string) (restore func() error, err error) { +func BackupFile(filePath, backupFileName string) (restore func() error, err error) { fileInfo, err := os.Stat(filePath) if err != nil { if os.IsNotExist(err) { - return createRestoreFileFunc(filePath, backupPath), nil + return createRestoreFileFunc(filePath, backupFileName), nil } return nil, errorutils.CheckError(err) } - fileMode := fileInfo.Mode() - if err = ioutils.CopyFile(filePath, backupPath, fileMode); err != nil { + if err = cloneFile(filePath, backupFileName, fileInfo.Mode()); err != nil { return nil, err } - log.Debug("The file", filePath, "was backed up successfully to", backupPath) - return createRestoreFileFunc(filePath, backupPath), nil + log.Debug("The file", filePath, "was backed up successfully to", backupFileName) + return createRestoreFileFunc(filePath, backupFileName), nil +} + +func cloneFile(origFile, newName string, fileMode os.FileMode) (err error) { + from, err := os.Open(origFile) + if errorutils.CheckError(err) != nil { + return + } + defer func() { + err = errors.Join(err, from.Close()) + }() + + to, err := os.OpenFile(filepath.Join(filepath.Dir(origFile), newName), os.O_RDWR|os.O_CREATE, fileMode) + if errorutils.CheckError(err) != nil { + return + } + defer func() { + err = errors.Join(err, to.Close()) + }() + + if _, err = io.Copy(to, from); err != nil { + err = errorutils.CheckError(err) + } + return } // createRestoreFileFunc creates a function for restoring a file from its backup. // The returned function replaces the file in filePath with the backup in backupPath. // If there is no file at backupPath (which means there was no file at filePath when BackupFile() was called), then the function deletes the file at filePath. -func createRestoreFileFunc(filePath, backupPath string) func() error { +func createRestoreFileFunc(filePath, backupFileName string) func() error { return func() error { + backupPath := filepath.Join(filepath.Dir(filePath), backupFileName) if _, err := os.Stat(backupPath); err != nil { if os.IsNotExist(err) { err = os.Remove(filePath) @@ -148,7 +173,6 @@ func createRestoreFileFunc(filePath, backupPath string) func() error { return errorutils.CheckError(err) } log.Debug("Restored the file", filePath, "successfully") - return nil } } diff --git a/artifactory/commands/utils/result_test.go b/artifactory/commands/utils/result_test.go index e7c5b66e7..81423a2ff 100644 --- a/artifactory/commands/utils/result_test.go +++ b/artifactory/commands/utils/result_test.go @@ -1,6 +1,7 @@ package utils import ( + biutils "github.com/jfrog/build-info-go/utils" testsutils "github.com/jfrog/jfrog-client-go/utils/tests" "os" "path" @@ -52,7 +53,7 @@ func createTempDeployableArtifactFile() (filePath string, err error) { if err != nil { return } - err = fileutils.CopyFile(tmpDir, summary.Name()) + err = biutils.CopyFile(tmpDir, summary.Name()) if err != nil { return } diff --git a/artifactory/commands/yarn/yarn.go b/artifactory/commands/yarn/yarn.go index 2d46b6dde..01a62713c 100644 --- a/artifactory/commands/yarn/yarn.go +++ b/artifactory/commands/yarn/yarn.go @@ -95,7 +95,7 @@ func (yc *YarnCommand) Run() error { }() } - restoreYarnrcFunc, err := commandUtils.BackupFile(filepath.Join(yc.workingDirectory, YarnrcFileName), filepath.Join(yc.workingDirectory, YarnrcBackupFileName)) + restoreYarnrcFunc, err := commandUtils.BackupFile(filepath.Join(yc.workingDirectory, YarnrcFileName), YarnrcBackupFileName) if err != nil { return RestoreConfigurationsAndError(nil, restoreYarnrcFunc, err) } diff --git a/artifactory/utils/buildutils_test.go b/artifactory/utils/buildutils_test.go index cb582db1b..3147c6be3 100644 --- a/artifactory/utils/buildutils_test.go +++ b/artifactory/utils/buildutils_test.go @@ -1,6 +1,7 @@ package utils import ( + biutils "github.com/jfrog/build-info-go/utils" "os" "path/filepath" "strconv" @@ -12,7 +13,6 @@ import ( "github.com/jfrog/jfrog-cli-core/v2/utils/coreutils" artclientutils "github.com/jfrog/jfrog-client-go/artifactory/services/utils" - "github.com/jfrog/jfrog-client-go/utils/io/fileutils" "github.com/stretchr/testify/assert" ) @@ -35,7 +35,7 @@ func TestGetBuildName(t *testing.T) { defer createTempDirCallback() confFileName := filepath.Join(tmpDir, ".jfrog", "projects") - assert.NoError(t, fileutils.CopyFile(confFileName, filepath.Join("testdata", "build.yaml"))) + assert.NoError(t, biutils.CopyFile(confFileName, filepath.Join("testdata", "build.yaml"))) wd, err := os.Getwd() assert.NoError(t, err, "Failed to get current dir") @@ -79,7 +79,7 @@ func TestGetEmptyBuildNameOnUnixAccessDenied(t *testing.T) { destConfFile := filepath.Join(tmpDir, ".jfrog", "projects") srcConfFile := filepath.Join("testdata", "build.yaml") - assert.NoError(t, fileutils.CopyFile(destConfFile, srcConfFile)) + assert.NoError(t, biutils.CopyFile(destConfFile, srcConfFile)) // Remove permissions from config file. assert.NoError(t, os.Chmod(destConfFile, 0000)) @@ -111,7 +111,7 @@ func TestGetBuildNumber(t *testing.T) { // Create build config in temp folder confFileName := filepath.Join(tmpDir, ".jfrog", "projects") - assert.NoError(t, fileutils.CopyFile(confFileName, filepath.Join("testdata", "build.yaml"))) + assert.NoError(t, biutils.CopyFile(confFileName, filepath.Join("testdata", "build.yaml"))) wd, err := os.Getwd() assert.NoError(t, err, "Failed to get current dir") @@ -217,7 +217,7 @@ func TestIsLoadedFromConfigFile(t *testing.T) { buildConfig.SetBuildName("") // Create build config in temp folder confFileName := filepath.Join(tmpDir, ".jfrog", "projects") - assert.NoError(t, fileutils.CopyFile(confFileName, filepath.Join("testdata", "build.yaml"))) + assert.NoError(t, biutils.CopyFile(confFileName, filepath.Join("testdata", "build.yaml"))) wd, err := os.Getwd() assert.NoError(t, err, "Failed to get current dir") chdirCallBack := testsutils.ChangeDirWithCallback(t, wd, tmpDir) diff --git a/artifactory/utils/dependenciesutils.go b/artifactory/utils/dependenciesutils.go index 7cfe5c4bf..790fd01fe 100644 --- a/artifactory/utils/dependenciesutils.go +++ b/artifactory/utils/dependenciesutils.go @@ -3,6 +3,7 @@ package utils import ( "errors" "fmt" + biutils "github.com/jfrog/build-info-go/utils" "github.com/jfrog/jfrog-cli-core/v2/utils/config" "github.com/jfrog/jfrog-cli-core/v2/utils/coreutils" xrayutils "github.com/jfrog/jfrog-cli-core/v2/xray/utils" @@ -224,7 +225,7 @@ func DownloadDependency(artDetails *config.ServerDetails, downloadPath, targetPa if err != nil { return err } - return fileutils.CopyDir(tempDirPath, localDir, true, nil) + return biutils.CopyDir(tempDirPath, localDir, true, nil) } func createHttpClient(artDetails *config.ServerDetails) (rtHttpClient *jfroghttpclient.JfrogHttpClient, httpClientDetails httputils.HttpClientDetails, err error) { diff --git a/go.mod b/go.mod index e102ff632..fffdf86a1 100644 --- a/go.mod +++ b/go.mod @@ -2,14 +2,16 @@ module github.com/jfrog/jfrog-cli-core/v2 go 1.20 +require github.com/c-bata/go-prompt v0.2.5 // Should not be updated to 0.2.6 due to a bug (https://github.com/jfrog/jfrog-cli-core/pull/372) + require ( github.com/buger/jsonparser v1.1.1 github.com/chzyer/readline v1.5.1 github.com/forPelevin/gomoji v1.1.8 github.com/gocarina/gocsv v0.0.0-20230616125104-99d496ca653d - github.com/google/uuid v1.3.0 + github.com/google/uuid v1.3.1 github.com/gookit/color v1.5.4 - github.com/jedib0t/go-pretty/v6 v6.4.6 + github.com/jedib0t/go-pretty/v6 v6.4.7 github.com/jfrog/build-info-go v1.9.9 github.com/jfrog/gofrog v1.3.0 github.com/jfrog/jfrog-client-go v1.31.6 @@ -27,20 +29,19 @@ require ( golang.org/x/term v0.11.0 golang.org/x/text v0.12.0 gopkg.in/yaml.v3 v3.0.1 -) -require github.com/c-bata/go-prompt v0.2.5 // Should not be updated to 0.2.6 due to a bug (https://github.com/jfrog/jfrog-cli-core/pull/372) +) require ( dario.cat/mergo v1.0.0 // indirect github.com/BurntSushi/toml v1.3.2 // indirect - github.com/CycloneDX/cyclonedx-go v0.7.1 // indirect + github.com/CycloneDX/cyclonedx-go v0.7.2 // indirect github.com/Microsoft/go-winio v0.6.1 // indirect - github.com/ProtonMail/go-crypto v0.0.0-20230717121422-5aa5874ade95 // indirect + github.com/ProtonMail/go-crypto v0.0.0-20230828082145-3c4c8a2d2371 // indirect github.com/VividCortex/ewma v1.2.0 // indirect github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d // indirect github.com/acomagu/bufpipe v1.0.4 // indirect - github.com/andybalholm/brotli v1.0.5 // indirect + github.com/andybalholm/brotli v1.0.1 // indirect github.com/cloudflare/circl v1.3.3 // indirect github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect github.com/davecgh/go-spew v1.1.1 // indirect @@ -52,15 +53,15 @@ require ( github.com/go-git/go-git/v5 v5.8.1 // indirect github.com/golang-jwt/jwt/v4 v4.5.0 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect - github.com/golang/snappy v0.0.4 // indirect + github.com/golang/snappy v0.0.2 // indirect github.com/hashicorp/hcl v1.0.0 // indirect github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect github.com/kevinburke/ssh_config v1.2.0 // indirect - github.com/klauspost/compress v1.15.9 // indirect + github.com/klauspost/compress v1.11.4 // indirect github.com/klauspost/cpuid/v2 v2.2.3 // indirect github.com/klauspost/pgzip v1.2.5 // indirect github.com/mattn/go-colorable v0.1.12 // indirect - github.com/mattn/go-isatty v0.0.16 // indirect + github.com/mattn/go-isatty v0.0.14 // indirect github.com/mattn/go-runewidth v0.0.13 // indirect github.com/mattn/go-tty v0.0.3 // indirect github.com/mholt/archiver/v3 v3.5.1 // indirect @@ -68,7 +69,7 @@ require ( github.com/mitchellh/mapstructure v1.5.0 // indirect github.com/nwaples/rardecode v1.1.0 // indirect github.com/pelletier/go-toml/v2 v2.0.8 // indirect - github.com/pierrec/lz4/v4 v4.1.15 // indirect + github.com/pierrec/lz4/v4 v4.1.2 // indirect github.com/pjbgf/sha1cd v0.3.0 // indirect github.com/pkg/term v1.1.0 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect @@ -86,15 +87,15 @@ require ( github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8 // indirect github.com/xo/terminfo v0.0.0-20210125001918-ca9a967f8778 // indirect golang.org/x/crypto v0.12.0 // indirect - golang.org/x/net v0.14.0 // indirect; indirectmake + golang.org/x/net v0.14.0 // indirect golang.org/x/sys v0.11.0 // indirect golang.org/x/tools v0.12.1-0.20230815132531-74c255bcf846 // indirect gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/warnings.v0 v0.1.2 // indirect ) -replace github.com/jfrog/jfrog-client-go => github.com/jfrog/jfrog-client-go v1.28.1-0.20230830130057-df2d2a80b555 +replace github.com/jfrog/jfrog-client-go => github.com/jfrog/jfrog-client-go v1.28.1-0.20230831152946-6ed2ae1aa57f -// replace github.com/jfrog/build-info-go => github.com/jfrog/build-info-go v1.8.9-0.20230828134416-f0db33dd9344 +replace github.com/jfrog/build-info-go => github.com/jfrog/build-info-go v1.8.9-0.20230831151231-e5e7bd035ddc // replace github.com/jfrog/gofrog => github.com/jfrog/gofrog v1.2.6-0.20230418122323-2bf299dd6d27 diff --git a/go.sum b/go.sum index faf2e4861..27735d28e 100644 --- a/go.sum +++ b/go.sum @@ -42,22 +42,21 @@ github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03 github.com/BurntSushi/toml v1.3.2 h1:o7IhLm0Msx3BaB+n3Ag7L8EVlByGnpq14C4YWiu/gL8= github.com/BurntSushi/toml v1.3.2/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= -github.com/CycloneDX/cyclonedx-go v0.7.1 h1:5w1SxjGm9MTMNTuRbEPyw21ObdbaagTWF/KfF0qHTRE= -github.com/CycloneDX/cyclonedx-go v0.7.1/go.mod h1:N/nrdWQI2SIjaACyyDs/u7+ddCkyl/zkNs8xFsHF2Ps= +github.com/CycloneDX/cyclonedx-go v0.7.2 h1:kKQ0t1dPOlugSIYVOMiMtFqeXI2wp/f5DBIdfux8gnQ= +github.com/CycloneDX/cyclonedx-go v0.7.2/go.mod h1:K2bA+324+Og0X84fA8HhN2X066K7Bxz4rpMQ4ZhjtSk= github.com/Microsoft/go-winio v0.5.2/go.mod h1:WpS1mjBmmwHBEWmogvA2mj8546UReBk4v8QkMxJ6pZY= github.com/Microsoft/go-winio v0.6.1 h1:9/kr64B9VUZrLm5YYwbGtUJnMgqWVOdUAXu6Migciow= github.com/Microsoft/go-winio v0.6.1/go.mod h1:LRdKpFKfdobln8UmuiYcKPot9D2v6svN5+sAH+4kjUM= -github.com/ProtonMail/go-crypto v0.0.0-20230717121422-5aa5874ade95 h1:KLq8BE0KwCL+mmXnjLWEAOYO+2l2AE4YMmqG1ZpZHBs= -github.com/ProtonMail/go-crypto v0.0.0-20230717121422-5aa5874ade95/go.mod h1:EjAoLdwvbIOoOQr3ihjnSoLZRtE8azugULFRteWMNc0= +github.com/ProtonMail/go-crypto v0.0.0-20230828082145-3c4c8a2d2371 h1:kkhsdkhsCvIsutKu5zLMgWtgh9YxGCNAw8Ad8hjwfYg= +github.com/ProtonMail/go-crypto v0.0.0-20230828082145-3c4c8a2d2371/go.mod h1:EjAoLdwvbIOoOQr3ihjnSoLZRtE8azugULFRteWMNc0= github.com/VividCortex/ewma v1.2.0 h1:f58SaIzcDXrSy3kWaHNvuJgJ3Nmz59Zji6XoJR/q1ow= github.com/VividCortex/ewma v1.2.0/go.mod h1:nz4BbCtbLyFDeC9SUHbtcT5644juEuWfUAUnGx7j5l4= github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d h1:licZJFw2RwpHMqeKTCYkitsPqHNxTmd4SNR5r94FGM8= github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d/go.mod h1:asat636LX7Bqt5lYEZ27JNDcqxfjdBQuJ/MM4CN/Lzo= github.com/acomagu/bufpipe v1.0.4 h1:e3H4WUzM3npvo5uv95QuJM3cQspFNtFBzvJ2oNjKIDQ= github.com/acomagu/bufpipe v1.0.4/go.mod h1:mxdxdup/WdsKVreO5GpW4+M/1CE2sMG4jeGJ2sYmHc4= +github.com/andybalholm/brotli v1.0.1 h1:KqhlKozYbRtJvsPrrEeXcO+N2l6NYT5A2QAFmSULpEc= github.com/andybalholm/brotli v1.0.1/go.mod h1:loMXtMfwqflxFJPmdbJO0a3KNoPuLBgiu3qAvBg8x/Y= -github.com/andybalholm/brotli v1.0.5 h1:8uQZIdzKmjc/iuPu7O2ioW48L81FgatrcpfFmiq/cCs= -github.com/andybalholm/brotli v1.0.5/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be h1:9AeTilPcZAjCFIImctFaOjnTIavg87rW78vTPkQqLI8= github.com/apparentlymart/go-textseg/v13 v13.0.0/go.mod h1:ZK2fH7c4NqDTLtiYLvIkEghdlcqw7yxLeM89kiTRPUo= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio= @@ -148,9 +147,8 @@ github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvq github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/snappy v0.0.2 h1:aeE13tS0IiQgFjYdoL8qN3K1N2bXXtI6Vi51/y7BpMw= github.com/golang/snappy v0.0.2/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= -github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= -github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= @@ -179,8 +177,8 @@ github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLe github.com/google/pprof v0.0.0-20201218002935-b9804c9f04c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= -github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.3.1 h1:KjJaJ9iWZ3jOFZIf1Lqf4laDRCasjl0BCmnEGxkdLb4= +github.com/google/uuid v1.3.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/googleapis/google-cloud-go-testing v0.0.0-20200911160855-bcd43fbb19e8/go.mod h1:dvDLG8qkwmyD9a/MJJN3XJcT3xFxOKAvTZGvuZmac9g= @@ -194,23 +192,22 @@ github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1: github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 h1:BQSFePA1RWJOlocH6Fxy8MmwDt+yVQYULKfN0RoTN8A= github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99/go.mod h1:1lJo3i6rXxKeerYnT8Nvf0QmHCRC1n8sfWVwXF2Frvo= -github.com/jedib0t/go-pretty/v6 v6.4.6 h1:v6aG9h6Uby3IusSSEjHaZNXpHFhzqMmjXcPq1Rjl9Jw= -github.com/jedib0t/go-pretty/v6 v6.4.6/go.mod h1:Ndk3ase2CkQbXLLNf5QDHoYb6J9WtVfmHZu9n8rk2xs= -github.com/jfrog/build-info-go v1.9.9 h1:YMA9okHawBNL8SrCWzqULSf5M4W+YnWyUhmkWSjoXEE= -github.com/jfrog/build-info-go v1.9.9/go.mod h1:t31QRpH5xUJKw8XkQlAA+Aq7aanyS1rrzpcK8xSNVts= +github.com/jedib0t/go-pretty/v6 v6.4.7 h1:lwiTJr1DEkAgzljsUsORmWsVn5MQjt1BPJdPCtJ6KXE= +github.com/jedib0t/go-pretty/v6 v6.4.7/go.mod h1:Ndk3ase2CkQbXLLNf5QDHoYb6J9WtVfmHZu9n8rk2xs= +github.com/jfrog/build-info-go v1.8.9-0.20230831151231-e5e7bd035ddc h1:pqu82clhPKyUKJcljMuxYa+kviaWnHycLNCLqZZNl30= +github.com/jfrog/build-info-go v1.8.9-0.20230831151231-e5e7bd035ddc/go.mod h1:QEskae5fQpjeY2PBzsjWtUQVskYSNDF2sSmw/Gx44dQ= github.com/jfrog/gofrog v1.3.0 h1:o4zgsBZE4QyDbz2M7D4K6fXPTBJht+8lE87mS9bw7Gk= github.com/jfrog/gofrog v1.3.0/go.mod h1:IFMc+V/yf7rA5WZ74CSbXe+Lgf0iApEQLxRZVzKRUR0= -github.com/jfrog/jfrog-client-go v1.28.1-0.20230830130057-df2d2a80b555 h1:yaF5J4LNk+ws5+j+BFMJ43NMqpKuCtF7dUfCeGLttl8= -github.com/jfrog/jfrog-client-go v1.28.1-0.20230830130057-df2d2a80b555/go.mod h1:icb00ZJN/mMMNkQduHDkzpqsXH9Flwi3f3COYexq3Nc= +github.com/jfrog/jfrog-client-go v1.28.1-0.20230831152946-6ed2ae1aa57f h1:S6l0o2sKFLRJ+QYVB5U/PJhrnwFSmKFFY7eHpRPRH8A= +github.com/jfrog/jfrog-client-go v1.28.1-0.20230831152946-6ed2ae1aa57f/go.mod h1:uUnMrqHX7Xi+OCaZEE4b3BtsmGeOSCB7XqaEWVXEH/E= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= github.com/kevinburke/ssh_config v1.2.0 h1:x584FjTGwHzMwvHx18PXxbBVzfnxogHaAReU4gf13a4= github.com/kevinburke/ssh_config v1.2.0/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.4.1/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= +github.com/klauspost/compress v1.11.4 h1:kz40R/YWls3iqT9zX9AHN3WoVsrAWVyui5sxuLqiXqU= github.com/klauspost/compress v1.11.4/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= -github.com/klauspost/compress v1.15.9 h1:wKRjX6JRtDdrE9qwa4b/Cip7ACOshUI4smpCQanqjSY= -github.com/klauspost/compress v1.15.9/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU= github.com/klauspost/cpuid v1.2.0/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= github.com/klauspost/cpuid/v2 v2.2.3 h1:sxCkb+qR91z4vsqw4vGGZlDgPz3G7gjaLyK3V8y70BU= github.com/klauspost/cpuid/v2 v2.2.3/go.mod h1:RVVoqg1df56z8g3pUjL/3lE5UfnlrJX8tyFgg4nqhuY= @@ -237,9 +234,8 @@ github.com/mattn/go-colorable v0.1.12/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.10/go.mod h1:qgIWMr58cqv1PHHyhnkY9lrL7etaEgOFcMEpPG5Rm84= github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= +github.com/mattn/go-isatty v0.0.14 h1:yVuAays6BHfxijgZPzw+3Zlu5yQgKGP2/hcQbHb7S9Y= github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= -github.com/mattn/go-isatty v0.0.16 h1:bq3VjFmv/sOjHtdEhmkEV4x1AJtvUvOJ2PFAZ5+peKQ= -github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= github.com/mattn/go-runewidth v0.0.6/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= github.com/mattn/go-runewidth v0.0.13 h1:lTGmDsbAYt5DmK6OnoV7EuIF1wEIFAcxld6ypU4OSgU= @@ -260,9 +256,8 @@ github.com/owenrumney/go-sarif/v2 v2.2.0 h1:1DmZaijK0HBZCR1fgcDSGa7VzYkU9NDmbZ7q github.com/owenrumney/go-sarif/v2 v2.2.0/go.mod h1:MSqMMx9WqlBSY7pXoOZWgEsVB4FDNfhcaXDA1j6Sr+w= github.com/pelletier/go-toml/v2 v2.0.8 h1:0ctb6s9mE31h0/lhu+J6OPmVeDxJn+kYnJc2jZR9tGQ= github.com/pelletier/go-toml/v2 v2.0.8/go.mod h1:vuYfssBdrU2XDZ9bYydBu6t+6a6PYNcZljzZR9VXg+4= +github.com/pierrec/lz4/v4 v4.1.2 h1:qvY3YFXRQE/XB8MlLzJH7mSzBs74eA2gg52YTk6jUPM= github.com/pierrec/lz4/v4 v4.1.2/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= -github.com/pierrec/lz4/v4 v4.1.15 h1:MO0/ucJhngq7299dKLwIMtgTfbkoSPF6AoMYDd8Q4q0= -github.com/pierrec/lz4/v4 v4.1.15/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= github.com/pjbgf/sha1cd v0.3.0 h1:4D5XXmUUBUl/xQ6IjCkEAbqXskkq/4O7LmGn0AqMDs4= github.com/pjbgf/sha1cd v0.3.0/go.mod h1:nZ1rrWOcGJ5uZgEEVL1VUM9iRQiZvWdbZjkKyFzPPsI= github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 h1:KoWmjvw+nsYOo29YJK9vDA65RGE3NrOnUtO7a+RF9HU= @@ -313,6 +308,7 @@ github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcU github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/subosito/gotenv v1.4.2 h1:X1TuBLAMDFbaTAChgCBLu3DU3UPyELpnF2jjJ2cz/S8= github.com/subosito/gotenv v1.4.2/go.mod h1:ayKnFf/c6rvx/2iiLrJUk1e6plDbT3edrFNGqEflhK0= +github.com/terminalstatic/go-xsd-validate v0.1.5 h1:RqpJnf6HGE2CB/lZB1A8BYguk8uRtcvYAPLCF15qguo= github.com/ulikunitz/xz v0.5.8/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= github.com/ulikunitz/xz v0.5.9 h1:RsKRIA2MO8x56wkkcd3LbtcE/uMszhb6DpRf+3uwa3I= github.com/ulikunitz/xz v0.5.9/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= @@ -324,6 +320,9 @@ github.com/vmihailenco/msgpack/v4 v4.3.12/go.mod h1:gborTTJjAo/GWTqqRjrLCn9pgNN+ github.com/vmihailenco/tagparser v0.1.1/go.mod h1:OeAg3pn3UbLjkWt+rN9oFYB6u/cQgqMEUPoW2WPyhdI= github.com/xanzy/ssh-agent v0.3.3 h1:+/15pJfg/RsTxqYcX6fHqOXZwwMP+2VyYWJeWM2qQFM= github.com/xanzy/ssh-agent v0.3.3/go.mod h1:6dzNDKs0J9rVPHPhaGCukekBHKqfl+L3KghI1Bc68Uw= +github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f h1:J9EGpcZtP0E/raorCMxlFGSTBrsSlaDGf3jU/qvAE2c= +github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 h1:EzJWgHovont7NscjpAxXsDA8S8BMYve8Y5+7cuRE7R0= +github.com/xeipuuv/gojsonschema v1.2.0 h1:LhYJRs+L4fBtjZUfuSZIKGeVu0QRy8e5Xi7D17UxZ74= github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8 h1:nIPpBwaJSVYIxUFsDv3M8ofmx9yWTog9BfvIu0q41lo= github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8/go.mod h1:HUYIGzjTL3rfEspMxjDjgmT5uz5wzYJKVo23qUhYTos= github.com/xo/terminfo v0.0.0-20210125001918-ca9a967f8778 h1:QldyIu/L63oPpyvQmHgvgickp1Yw510KJOqX7H24mg8= @@ -505,7 +504,6 @@ golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220704084225-05e143d24a9e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220909162455-aba9fc2a8ff2/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= diff --git a/utils/config/config.go b/utils/config/config.go index 019f979b6..af1774c72 100644 --- a/utils/config/config.go +++ b/utils/config/config.go @@ -5,6 +5,7 @@ import ( "encoding/json" "errors" "github.com/buger/jsonparser" + biutils "github.com/jfrog/build-info-go/utils" "github.com/jfrog/jfrog-cli-core/v2/utils/coreutils" cliLog "github.com/jfrog/jfrog-cli-core/v2/utils/log" accessAuth "github.com/jfrog/jfrog-client-go/access/auth" @@ -390,7 +391,7 @@ func createHomeDirBackup() error { curBackupPath := filepath.Join(backupDir, backupName) log.Debug("Creating a homedir backup at: " + curBackupPath) exclude := []string{coreutils.JfrogBackupDirName, coreutils.JfrogDependenciesDirName, coreutils.JfrogLocksDirName, coreutils.JfrogLogsDirName} - return fileutils.CopyDir(homeDir, curBackupPath, true, exclude) + return biutils.CopyDir(homeDir, curBackupPath, true, exclude) } // Version key doesn't exist in version 0 diff --git a/utils/config/tests/utils.go b/utils/config/tests/utils.go index 0a62cf8ba..1bde9e610 100644 --- a/utils/config/tests/utils.go +++ b/utils/config/tests/utils.go @@ -1,8 +1,8 @@ package tests import ( + biutils "github.com/jfrog/build-info-go/utils" "github.com/jfrog/jfrog-cli-core/v2/utils/coreutils" - "github.com/jfrog/jfrog-client-go/utils/io/fileutils" testsutils "github.com/jfrog/jfrog-client-go/utils/tests" "github.com/stretchr/testify/assert" "os" @@ -29,5 +29,5 @@ func CreateTempEnv(t *testing.T, copyEncryptionKey bool) (cleanUp func()) { } func copyResources(t *testing.T, sourcePath string, destPath string) { - assert.NoError(t, fileutils.CopyDir(sourcePath, destPath, true, nil)) + assert.NoError(t, biutils.CopyDir(sourcePath, destPath, true, nil)) } diff --git a/utils/ioutils/ioutils.go b/utils/ioutils/ioutils.go index 05ca35d97..5978302d2 100644 --- a/utils/ioutils/ioutils.go +++ b/utils/ioutils/ioutils.go @@ -7,7 +7,6 @@ import ( "github.com/jfrog/jfrog-client-go/utils/errorutils" "github.com/jfrog/jfrog-client-go/utils/log" "golang.org/x/term" - "io" "os" "strings" "syscall" @@ -66,36 +65,6 @@ func ScanFromConsole(caption string, scanInto *string, defaultValue string) { *scanInto = strings.TrimSpace(*scanInto) } -func CopyFile(src, dst string, fileMode os.FileMode) (err error) { - from, err := os.Open(src) - if err != nil { - return errorutils.CheckError(err) - } - defer func() { - e := from.Close() - if err == nil { - err = e - } - }() - - to, err := os.OpenFile(dst, os.O_RDWR|os.O_CREATE, fileMode) - if err != nil { - return errorutils.CheckError(err) - } - defer func() { - e := to.Close() - if err == nil { - err = e - } - }() - - if _, err = io.Copy(to, from); err != nil { - return errorutils.CheckError(err) - } - - return errorutils.CheckError(os.Chmod(dst, fileMode)) -} - func DoubleWinPathSeparator(filePath string) string { return strings.ReplaceAll(filePath, "\\", "\\\\") } diff --git a/utils/plugins/utils_test.go b/utils/plugins/utils_test.go index 1eb94bac6..30579d244 100644 --- a/utils/plugins/utils_test.go +++ b/utils/plugins/utils_test.go @@ -2,6 +2,7 @@ package plugins import ( "fmt" + biutils "github.com/jfrog/build-info-go/utils" "github.com/jfrog/jfrog-cli-core/v2/utils/coreutils" "github.com/jfrog/jfrog-client-go/utils/io/fileutils" testsutils "github.com/jfrog/jfrog-client-go/utils/tests" @@ -88,7 +89,7 @@ func setupPluginsTestingEnv(t *testing.T, pluginsDirName string) string { assert.NoError(t, err) wd, err := os.Getwd() assert.NoError(t, err) - err = fileutils.CopyDir(filepath.Join(wd, "testdata", coreutils.JfrogPluginsDirName, pluginsDirName), filepath.Join(testHomeDir, coreutils.JfrogPluginsDirName), true, nil) + err = biutils.CopyDir(filepath.Join(wd, "testdata", coreutils.JfrogPluginsDirName, pluginsDirName), filepath.Join(testHomeDir, coreutils.JfrogPluginsDirName), true, nil) assert.NoError(t, err) err = coreutils.ChmodPluginsDirectoryContent() assert.NoError(t, err) diff --git a/utils/python/utils_test.go b/utils/python/utils_test.go index 6e188d305..3b88edc0d 100644 --- a/utils/python/utils_test.go +++ b/utils/python/utils_test.go @@ -1,6 +1,7 @@ package utils import ( + biutils "github.com/jfrog/build-info-go/utils" "github.com/jfrog/jfrog-cli-core/v2/xray/audit" "github.com/jfrog/jfrog-client-go/utils/io/fileutils" "github.com/stretchr/testify/assert" @@ -28,6 +29,6 @@ func initPoetryTest(t *testing.T) (string, func()) { testAbs, err := filepath.Abs(filepath.Join("..", "..", "xray", "commands", "testdata", "poetry-project")) assert.NoError(t, err) poetryProjectPath, cleanUp := audit.CreateTestWorkspace(t, "poetry-project") - assert.NoError(t, fileutils.CopyDir(testAbs, poetryProjectPath, true, nil)) + assert.NoError(t, biutils.CopyDir(testAbs, poetryProjectPath, true, nil)) return poetryProjectPath, cleanUp } diff --git a/utils/tests/utils.go b/utils/tests/utils.go index 0b8acfe3f..3211255bb 100644 --- a/utils/tests/utils.go +++ b/utils/tests/utils.go @@ -125,15 +125,15 @@ func compare(expected, actual []string) error { } // CompareTree returns true iff the two trees contain the same nodes (regardless of their order) -func CompareTree(a, b *xrayUtils.GraphNode) bool { - if a.Id != b.Id { +func CompareTree(expected, actual *xrayUtils.GraphNode) bool { + if expected.Id != actual.Id { return false } // Make sure all children are equal, when order doesn't matter - for _, nodeA := range a.Nodes { + for _, expectedNode := range expected.Nodes { found := false - for _, nodeB := range b.Nodes { - if CompareTree(nodeA, nodeB) { + for _, actualNode := range actual.Nodes { + if CompareTree(expectedNode, actualNode) { found = true break } diff --git a/xray/audit/commonutils.go b/xray/audit/commonutils.go index b2f79997b..76751af9a 100644 --- a/xray/audit/commonutils.go +++ b/xray/audit/commonutils.go @@ -2,89 +2,80 @@ package audit import ( "fmt" + biutils "github.com/jfrog/build-info-go/utils" "github.com/jfrog/jfrog-cli-core/v2/utils/coreutils" - xraycommands "github.com/jfrog/jfrog-cli-core/v2/xray/commands/utils" - xrayUtils "github.com/jfrog/jfrog-client-go/xray/services/utils" - "os" - "os/exec" - "path/filepath" - "strings" - "testing" - - buildinfo "github.com/jfrog/build-info-go/entities" "github.com/jfrog/jfrog-cli-core/v2/utils/tests" + xraycommands "github.com/jfrog/jfrog-cli-core/v2/xray/commands/utils" "github.com/jfrog/jfrog-client-go/utils/errorutils" ioUtils "github.com/jfrog/jfrog-client-go/utils/io" - "github.com/jfrog/jfrog-client-go/utils/io/fileutils" "github.com/jfrog/jfrog-client-go/utils/log" testsutils "github.com/jfrog/jfrog-client-go/utils/tests" "github.com/jfrog/jfrog-client-go/xray/services" + xrayUtils "github.com/jfrog/jfrog-client-go/xray/services/utils" "github.com/stretchr/testify/assert" - "golang.org/x/exp/slices" + "golang.org/x/exp/maps" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" ) -func BuildXrayDependencyTree(treeHelper map[string][]string, nodeId string) *xrayUtils.GraphNode { - exceededDepthCounter := 0 - xrayDependencyTree := buildXrayDependencyTree(treeHelper, []string{nodeId}, &exceededDepthCounter) - if exceededDepthCounter > 0 { - log.Debug("buildXrayDependencyTree exceeded max tree depth", exceededDepthCounter, "times") +const maxUniqueAppearances = 10 + +func BuildXrayDependencyTree(treeHelper map[string][]string, nodeId string) (*xrayUtils.GraphNode, []string) { + rootNode := &xrayUtils.GraphNode{ + Id: nodeId, + Nodes: []*xrayUtils.GraphNode{}, } - return xrayDependencyTree + dependencyAppearances := map[string]int8{} + populateXrayDependencyTree(rootNode, treeHelper, &dependencyAppearances) + return rootNode, maps.Keys(dependencyAppearances) } -func buildXrayDependencyTree(treeHelper map[string][]string, impactPath []string, exceededDepthCounter *int) *xrayUtils.GraphNode { - nodeId := impactPath[len(impactPath)-1] - // Initialize the new node - xrDependencyTree := &xrayUtils.GraphNode{} - xrDependencyTree.Id = nodeId - xrDependencyTree.Nodes = []*xrayUtils.GraphNode{} - if len(impactPath) >= buildinfo.RequestedByMaxLength { - *exceededDepthCounter++ - return xrDependencyTree - } +func populateXrayDependencyTree(currNode *xrayUtils.GraphNode, treeHelper map[string][]string, dependencyAppearances *map[string]int8) { + (*dependencyAppearances)[currNode.Id]++ // Recursively create & append all node's dependencies. - for _, dependency := range treeHelper[nodeId] { - // Prevent circular dependencies parsing - if slices.Contains(impactPath, dependency) { + for _, childDepId := range treeHelper[currNode.Id] { + childNode := &xrayUtils.GraphNode{ + Id: childDepId, + Nodes: []*xrayUtils.GraphNode{}, + Parent: currNode, + } + if (*dependencyAppearances)[childDepId] >= maxUniqueAppearances || childNode.NodeHasLoop() { continue } - xrDependencyTree.Nodes = append(xrDependencyTree.Nodes, buildXrayDependencyTree(treeHelper, append(impactPath, dependency), exceededDepthCounter)) + currNode.Nodes = append(currNode.Nodes, childNode) + populateXrayDependencyTree(childNode, treeHelper, dependencyAppearances) } - return xrDependencyTree } -func RunXrayDependenciesTreeScanGraph(modulesDependencyTrees []*xrayUtils.GraphNode, progress ioUtils.ProgressMgr, technology coreutils.Technology, scanGraphParams *xraycommands.ScanGraphParams) (results []services.ScanResponse, err error) { +func RunXrayDependenciesTreeScanGraph(dependencyTree *xrayUtils.GraphNode, progress ioUtils.ProgressMgr, technology coreutils.Technology, scanGraphParams *xraycommands.ScanGraphParams) (results []services.ScanResponse, err error) { + scanGraphParams.XrayGraphScanParams().DependenciesGraph = dependencyTree + scanMessage := fmt.Sprintf("Scanning %d %s dependencies", len(dependencyTree.Nodes), technology) if progress != nil { - progress.SetHeadlineMsg("Scanning for vulnerabilities") + progress.SetHeadlineMsg(scanMessage) } - - for _, moduleDependencyTree := range modulesDependencyTrees { - scanGraphParams.XrayGraphScanParams().Graph = moduleDependencyTree - scanMessage := fmt.Sprintf("Scanning %d %s dependencies", len(scanGraphParams.XrayGraphScanParams().Graph.Nodes), technology) - if progress != nil { - progress.SetHeadlineMsg(scanMessage) - } - log.Info(scanMessage + "...") - var scanResults *services.ScanResponse - scanResults, err = xraycommands.RunScanGraphAndGetResults(scanGraphParams) - if err != nil { - err = errorutils.CheckErrorf("scanning %s dependencies failed with error: %s", string(technology), err.Error()) - return - } - for i := range scanResults.Vulnerabilities { - scanResults.Vulnerabilities[i].Technology = technology.ToString() - } - for i := range scanResults.Violations { - scanResults.Violations[i].Technology = technology.ToString() - } - results = append(results, *scanResults) + log.Info(scanMessage + "...") + var scanResults *services.ScanResponse + scanResults, err = xraycommands.RunScanGraphAndGetResults(scanGraphParams) + if err != nil { + err = errorutils.CheckErrorf("scanning %s dependencies failed with error: %s", string(technology), err.Error()) + return + } + for i := range scanResults.Vulnerabilities { + scanResults.Vulnerabilities[i].Technology = technology.ToString() + } + for i := range scanResults.Violations { + scanResults.Violations[i].Technology = technology.ToString() } + results = append(results, *scanResults) return } func CreateTestWorkspace(t *testing.T, sourceDir string) (string, func()) { tempDirPath, createTempDirCallback := tests.CreateTempDirWithCallbackAndAssert(t) - assert.NoError(t, fileutils.CopyDir(filepath.Join("..", "..", "commands", "testdata", sourceDir), tempDirPath, true, nil)) + assert.NoError(t, biutils.CopyDir(filepath.Join("..", "..", "commands", "testdata", sourceDir), tempDirPath, true, nil)) wd, err := os.Getwd() assert.NoError(t, err, "Failed to get current dir") chdirCallback := testsutils.ChangeDirWithCallback(t, wd, tempDirPath) diff --git a/xray/audit/go/gloang_test.go b/xray/audit/go/gloang_test.go index 82e0aced8..4d0b29fcf 100644 --- a/xray/audit/go/gloang_test.go +++ b/xray/audit/go/gloang_test.go @@ -32,8 +32,19 @@ func TestBuildGoDependencyList(t *testing.T) { User: "user", AccessToken: "sdsdccs2232", } - rootNode, err := BuildDependencyTree(server, "test-remote") + goVersionID, err := getGoVersionAsDependency() assert.NoError(t, err) + expectedUniqueDeps := []string{ + goPackageTypeIdentifier + "golang.org/x/text:v0.3.3", + goPackageTypeIdentifier + "rsc.io/quote:v1.5.2", + goPackageTypeIdentifier + "rsc.io/sampler:v1.3.0", + goPackageTypeIdentifier + "testGoList", + goVersionID.Id, + } + rootNode, uniqueDeps, err := BuildDependencyTree(server, "test-remote") + assert.NoError(t, err) + assert.ElementsMatch(t, uniqueDeps, expectedUniqueDeps, "First is actual, Second is Expected") + assert.Equal(t, "https://user:sdsdccs2232@api.go.here/artifactoryapi/go/test-remote|direct", os.Getenv("GOPROXY")) assert.NotEmpty(t, rootNode) diff --git a/xray/audit/go/golang.go b/xray/audit/go/golang.go index f6d0010fa..86c35fc12 100644 --- a/xray/audit/go/golang.go +++ b/xray/audit/go/golang.go @@ -2,6 +2,7 @@ package _go import ( "github.com/jfrog/build-info-go/utils" + "github.com/jfrog/gofrog/datastructures" "github.com/jfrog/jfrog-cli-core/v2/utils/config" "os" "strings" @@ -16,7 +17,7 @@ const ( goSourceCodePrefix = "github.com/golang/go:v" ) -func BuildDependencyTree(server *config.ServerDetails, remoteGoRepo string) (dependencyTree []*xrayUtils.GraphNode, err error) { +func BuildDependencyTree(server *config.ServerDetails, remoteGoRepo string) (dependencyTree []*xrayUtils.GraphNode, uniqueDeps []string, err error) { currentDir, err := coreutils.GetWorkingDirectory() if err != nil { return @@ -46,15 +47,18 @@ func BuildDependencyTree(server *config.ServerDetails, remoteGoRepo string) (dep Id: goPackageTypeIdentifier + rootModuleName, Nodes: []*xrayUtils.GraphNode{}, } - populateGoDependencyTree(rootNode, dependenciesGraph, dependenciesList) + uniqueDepsSet := datastructures.MakeSet[string]() + populateGoDependencyTree(rootNode, dependenciesGraph, dependenciesList, uniqueDepsSet) - // Add go version as child node to dependencies tree - err = addGoVersionAsDependency(rootNode) + goVersionDependency, err := getGoVersionAsDependency() if err != nil { return } + rootNode.Nodes = append(rootNode.Nodes, goVersionDependency) + uniqueDepsSet.Add(goVersionDependency.Id) dependencyTree = []*xrayUtils.GraphNode{rootNode} + uniqueDeps = uniqueDepsSet.ToSlice() return } @@ -67,10 +71,11 @@ func setGoProxy(server *config.ServerDetails, remoteGoRepo string) error { return os.Setenv("GOPROXY", repoUrl) } -func populateGoDependencyTree(currNode *xrayUtils.GraphNode, dependenciesGraph map[string][]string, dependenciesList map[string]bool) { +func populateGoDependencyTree(currNode *xrayUtils.GraphNode, dependenciesGraph map[string][]string, dependenciesList map[string]bool, uniqueDepsSet *datastructures.Set[string]) { if currNode.NodeHasLoop() { return } + uniqueDepsSet.Add(currNode.Id) currDepChildren := dependenciesGraph[strings.TrimPrefix(currNode.Id, goPackageTypeIdentifier)] // Recursively create & append all node's dependencies. for _, childName := range currDepChildren { @@ -84,20 +89,18 @@ func populateGoDependencyTree(currNode *xrayUtils.GraphNode, dependenciesGraph m Parent: currNode, } currNode.Nodes = append(currNode.Nodes, childNode) - populateGoDependencyTree(childNode, dependenciesGraph, dependenciesList) + populateGoDependencyTree(childNode, dependenciesGraph, dependenciesList, uniqueDepsSet) } } -func addGoVersionAsDependency(rootNode *xrayUtils.GraphNode) error { +func getGoVersionAsDependency() (*xrayUtils.GraphNode, error) { goVersion, err := utils.GetParsedGoVersion() if err != nil { - return err + return nil, err } // Convert "go1.17.3" to "github.com/golang/go:v1.17.3" goVersionID := strings.ReplaceAll(goVersion.GetVersion(), "go", goSourceCodePrefix) - rootNode.Nodes = append(rootNode.Nodes, &xrayUtils.GraphNode{ - Id: goPackageTypeIdentifier + goVersionID, - Nodes: []*xrayUtils.GraphNode{}, - }) - return nil + return &xrayUtils.GraphNode{ + Id: goPackageTypeIdentifier + goVersionID, + }, nil } diff --git a/xray/audit/java/gradle.go b/xray/audit/java/gradle.go index 5db177045..d140efffd 100644 --- a/xray/audit/java/gradle.go +++ b/xray/audit/java/gradle.go @@ -6,6 +6,7 @@ import ( "encoding/json" "errors" "fmt" + "github.com/jfrog/gofrog/datastructures" "os" "os/exec" "path/filepath" @@ -105,11 +106,13 @@ func (dtp *depTreeManager) appendDependenciesPaths(jsonDepTree []byte, fileName if dtp.tree == nil { dtp.tree = make(map[string][]dependenciesPaths) } - dtp.tree[fileName] = append(dtp.tree[fileName], deps) + if len(deps.Paths) > 0 { + dtp.tree[fileName] = append(dtp.tree[fileName], deps) + } return nil } -func buildGradleDependencyTree(params *DependencyTreeParams) (dependencyTree []*xrayUtils.GraphNode, err error) { +func buildGradleDependencyTree(params *DependencyTreeParams) (dependencyTree []*xrayUtils.GraphNode, uniqueDeps []string, err error) { manager := &depTreeManager{useWrapper: params.UseWrapper} if params.IgnoreConfigFile { // In case we don't need to use the gradle config file, @@ -125,9 +128,10 @@ func buildGradleDependencyTree(params *DependencyTreeParams) (dependencyTree []* outputFileContent, err := manager.runGradleDepTree() if err != nil { - return nil, err + return } - return manager.getGraphFromDepTree(outputFileContent) + dependencyTree, uniqueDeps, err = manager.getGraphFromDepTree(outputFileContent) + return } func (dtp *depTreeManager) runGradleDepTree() (outputFileContent []byte, err error) { @@ -234,25 +238,27 @@ func (dtp *depTreeManager) execGradleDepTree(depTreeDir string) (outputFileConte } // Assuming we ran gradle-dep-tree, getGraphFromDepTree receives the content of the depTreeOutputFile as input -func (dtp *depTreeManager) getGraphFromDepTree(outputFileContent []byte) ([]*xrayUtils.GraphNode, error) { +func (dtp *depTreeManager) getGraphFromDepTree(outputFileContent []byte) ([]*xrayUtils.GraphNode, []string, error) { if err := dtp.parseDepTreeFiles(outputFileContent); err != nil { - return nil, err + return nil, nil, err } var depsGraph []*xrayUtils.GraphNode + uniqueDepsSet := datastructures.MakeSet[string]() for dependency, children := range dtp.tree { directDependency := &xrayUtils.GraphNode{ Id: GavPackageTypeIdentifier + dependency, Nodes: []*xrayUtils.GraphNode{}, } for _, childPath := range children { - populateGradleDependencyTree(directDependency, childPath) + populateGradleDependencyTree(directDependency, childPath, uniqueDepsSet) } depsGraph = append(depsGraph, directDependency) } - return depsGraph, nil + return depsGraph, uniqueDepsSet.ToSlice(), nil } -func populateGradleDependencyTree(currNode *xrayUtils.GraphNode, currNodeChildren dependenciesPaths) { +func populateGradleDependencyTree(currNode *xrayUtils.GraphNode, currNodeChildren dependenciesPaths, uniqueDepsSet *datastructures.Set[string]) { + uniqueDepsSet.Add(currNode.Id) for gav, children := range currNodeChildren.Paths { childNode := &xrayUtils.GraphNode{ Id: GavPackageTypeIdentifier + gav, @@ -262,7 +268,7 @@ func populateGradleDependencyTree(currNode *xrayUtils.GraphNode, currNodeChildre if currNode.NodeHasLoop() { return } - populateGradleDependencyTree(childNode, children) + populateGradleDependencyTree(childNode, children, uniqueDepsSet) currNode.Nodes = append(currNode.Nodes, childNode) } } diff --git a/xray/audit/java/gradle_test.go b/xray/audit/java/gradle_test.go index 591e7ef88..241c5a1b1 100644 --- a/xray/audit/java/gradle_test.go +++ b/xray/audit/java/gradle_test.go @@ -46,9 +46,10 @@ func TestGradleTreesWithoutConfig(t *testing.T) { assert.NoError(t, os.Chmod(filepath.Join(tempDirPath, "gradlew"), 0700)) // Run getModulesDependencyTrees - modulesDependencyTrees, err := buildGradleDependencyTree(&DependencyTreeParams{}) + modulesDependencyTrees, uniqueDeps, err := buildGradleDependencyTree(&DependencyTreeParams{}) if assert.NoError(t, err) && assert.NotNil(t, modulesDependencyTrees) { - assert.Len(t, modulesDependencyTrees, 5) + assert.Len(t, uniqueDeps, 11) + assert.Len(t, modulesDependencyTrees, 2) // Check module module := audit.GetAndAssertNode(t, modulesDependencyTrees, "webservice") assert.Len(t, module.Nodes, 7) @@ -69,10 +70,10 @@ func TestGradleTreesWithConfig(t *testing.T) { assert.NoError(t, os.Chmod(filepath.Join(tempDirPath, "gradlew"), 0700)) // Run getModulesDependencyTrees - modulesDependencyTrees, err := buildGradleDependencyTree(&DependencyTreeParams{UseWrapper: true}) + modulesDependencyTrees, uniqueDeps, err := buildGradleDependencyTree(&DependencyTreeParams{UseWrapper: true}) if assert.NoError(t, err) && assert.NotNil(t, modulesDependencyTrees) { - assert.Len(t, modulesDependencyTrees, 5) - + assert.Len(t, modulesDependencyTrees, 3) + assert.Len(t, uniqueDeps, 11) // Check module module := audit.GetAndAssertNode(t, modulesDependencyTrees, "api") assert.Len(t, module.Nodes, 4) @@ -93,13 +94,12 @@ func TestGradleTreesExcludeTestDeps(t *testing.T) { assert.NoError(t, os.Chmod(filepath.Join(tempDirPath, "gradlew"), 0700)) // Run getModulesDependencyTrees - modulesDependencyTrees, err := buildGradleDependencyTree(&DependencyTreeParams{UseWrapper: true}) + modulesDependencyTrees, uniqueDeps, err := buildGradleDependencyTree(&DependencyTreeParams{UseWrapper: true}) if assert.NoError(t, err) && assert.NotNil(t, modulesDependencyTrees) { - assert.Len(t, modulesDependencyTrees, 5) - + assert.Len(t, modulesDependencyTrees, 2) + assert.Len(t, uniqueDeps, 11) // Check direct dependency - directDependency := audit.GetAndAssertNode(t, modulesDependencyTrees, "services") - assert.Empty(t, directDependency.Nodes) + assert.Nil(t, audit.GetModule(modulesDependencyTrees, "services")) } } @@ -177,11 +177,12 @@ func TestGetGraphFromDepTree(t *testing.T) { }() assert.NoError(t, os.Chmod(filepath.Join(tempDirPath, "gradlew"), 0700)) testCase := struct { - name string - expectedResult map[string]map[string]string + name string + expectedTree map[string]map[string]string + expectedUniqueDeps []string }{ name: "ValidOutputFileContent", - expectedResult: map[string]map[string]string{ + expectedTree: map[string]map[string]string{ GavPackageTypeIdentifier + "shared": {}, GavPackageTypeIdentifier + filepath.Base(tempDirPath): {}, GavPackageTypeIdentifier + "services": {}, @@ -200,15 +201,30 @@ func TestGetGraphFromDepTree(t *testing.T) { GavPackageTypeIdentifier + "commons-lang:commons-lang:2.4": "", }, }, + expectedUniqueDeps: []string{ + GavPackageTypeIdentifier + "webservice", + GavPackageTypeIdentifier + "junit:junit:4.11", + GavPackageTypeIdentifier + "commons-io:commons-io:1.2", + GavPackageTypeIdentifier + "org.apache.wicket:wicket:1.3.7", + GavPackageTypeIdentifier + "org.jfrog.example.gradle:shared:1.0", + GavPackageTypeIdentifier + "org.jfrog.example.gradle:api:1.0", + GavPackageTypeIdentifier + "commons-collections:commons-collections:3.2", + GavPackageTypeIdentifier + "api", + GavPackageTypeIdentifier + "commons-lang:commons-lang:2.4", + GavPackageTypeIdentifier + "org.hamcrest:hamcrest-core:1.3", + GavPackageTypeIdentifier + "org.slf4j:slf4j-api:1.4.2", + }, } manager := &depTreeManager{} outputFileContent, err := manager.runGradleDepTree() assert.NoError(t, err) - result, err := (&depTreeManager{}).getGraphFromDepTree(outputFileContent) + depTree, uniqueDeps, err := (&depTreeManager{}).getGraphFromDepTree(outputFileContent) assert.NoError(t, err) - for _, dependency := range result { - depChild, exists := testCase.expectedResult[dependency.Id] + assert.ElementsMatch(t, uniqueDeps, testCase.expectedUniqueDeps, "First is actual, Second is Expected") + + for _, dependency := range depTree { + depChild, exists := testCase.expectedTree[dependency.Id] assert.True(t, exists) assert.Equal(t, len(depChild), len(dependency.Nodes)) } diff --git a/xray/audit/java/javautils.go b/xray/audit/java/javautils.go index 93421aaf1..866b779af 100644 --- a/xray/audit/java/javautils.go +++ b/xray/audit/java/javautils.go @@ -1,8 +1,10 @@ package java import ( + "github.com/jfrog/gofrog/datastructures" "github.com/jfrog/jfrog-cli-core/v2/utils/config" "github.com/jfrog/jfrog-cli-core/v2/utils/coreutils" + xrayutils "github.com/jfrog/jfrog-cli-core/v2/xray/utils" xrayUtils "github.com/jfrog/jfrog-client-go/xray/services/utils" "strconv" "time" @@ -40,35 +42,36 @@ func createBuildConfiguration(buildName string) (*artifactoryUtils.BuildConfigur // Create a dependency tree for each one of the modules in the build. // buildName - audit-mvn or audit-gradle -func createGavDependencyTree(buildConfig *artifactoryUtils.BuildConfiguration) ([]*xrayUtils.GraphNode, error) { +func createGavDependencyTree(buildConfig *artifactoryUtils.BuildConfiguration) ([]*xrayUtils.GraphNode, []string, error) { buildName, err := buildConfig.GetBuildName() if err != nil { - return nil, err + return nil, nil, err } buildNumber, err := buildConfig.GetBuildNumber() if err != nil { - return nil, err + return nil, nil, err } generatedBuildsInfos, err := artifactoryUtils.GetGeneratedBuildsInfo(buildName, buildNumber, buildConfig.GetProject()) if err != nil { - return nil, err + return nil, nil, err } if len(generatedBuildsInfos) == 0 { - return nil, errorutils.CheckErrorf("Couldn't find build " + buildName + "/" + buildNumber) + return nil, nil, errorutils.CheckErrorf("Couldn't find build " + buildName + "/" + buildNumber) } modules := []*xrayUtils.GraphNode{} + uniqueDepsSet := datastructures.MakeSet[string]() for _, module := range generatedBuildsInfos[0].Modules { - modules = append(modules, addModuleTree(module)) + modules = append(modules, addModuleTree(module, uniqueDepsSet)) } - return modules, nil + return modules, uniqueDepsSet.ToSlice(), nil } -func addModuleTree(module buildinfo.Module) *xrayUtils.GraphNode { +func addModuleTree(module buildinfo.Module, uniqueDepsSet *datastructures.Set[string]) *xrayUtils.GraphNode { moduleTree := &xrayUtils.GraphNode{ Id: GavPackageTypeIdentifier + module.Id, } - + uniqueDepsSet.Add(moduleTree.Id) directDependencies := make(map[string]buildinfo.Dependency) parentToChildren := newDependencyMultimap() for index, dependency := range module.Dependencies { @@ -86,7 +89,7 @@ func addModuleTree(module buildinfo.Module) *xrayUtils.GraphNode { } for _, directDependency := range directDependencies { - populateTransitiveDependencies(moduleTree, directDependency.Id, parentToChildren, []string{}) + populateTransitiveDependencies(moduleTree, directDependency.Id, parentToChildren, []string{}, uniqueDepsSet) } return moduleTree } @@ -105,7 +108,7 @@ func isDirectDependency(moduleId string, requestedBy [][]string) bool { return false } -func populateTransitiveDependencies(parent *xrayUtils.GraphNode, dependencyId string, parentToChildren *dependencyMultimap, idsAdded []string) { +func populateTransitiveDependencies(parent *xrayUtils.GraphNode, dependencyId string, parentToChildren *dependencyMultimap, idsAdded []string, uniqueDepsSet *datastructures.Set[string]) { if hasLoop(idsAdded, dependencyId) { return } @@ -114,9 +117,10 @@ func populateTransitiveDependencies(parent *xrayUtils.GraphNode, dependencyId st Id: GavPackageTypeIdentifier + dependencyId, Nodes: []*xrayUtils.GraphNode{}, } + uniqueDepsSet.Add(node.Id) parent.Nodes = append(parent.Nodes, node) for _, child := range parentToChildren.getChildren(node.Id) { - populateTransitiveDependencies(node, child.Id, parentToChildren, idsAdded) + populateTransitiveDependencies(node, child.Id, parentToChildren, idsAdded, uniqueDepsSet) } } @@ -129,11 +133,24 @@ func hasLoop(idsAdded []string, idToAdd string) bool { return false } -func BuildDependencyTree(params *DependencyTreeParams) (modules []*xrayUtils.GraphNode, err error) { - if params.Tool == coreutils.Maven { - return buildMvnDependencyTree(params) +func BuildDependencyTree(params *xrayutils.GraphBasicParams, tech coreutils.Technology) ([]*xrayUtils.GraphNode, []string, error) { + serverDetails, err := params.ServerDetails() + if err != nil { + return nil, nil, err + } + dependencyTreeParams := &DependencyTreeParams{ + Tool: tech, + InsecureTls: params.InsecureTls(), + IgnoreConfigFile: params.IgnoreConfigFile(), + ExcludeTestDeps: params.ExcludeTestDependencies(), + UseWrapper: params.UseWrapper(), + Server: serverDetails, + DepsRepo: params.DepsRepo(), + } + if tech == coreutils.Maven { + return buildMvnDependencyTree(dependencyTreeParams) } - return buildGradleDependencyTree(params) + return buildGradleDependencyTree(dependencyTreeParams) } type dependencyMultimap struct { diff --git a/xray/audit/java/mvn.go b/xray/audit/java/mvn.go index f4039d585..554457377 100644 --- a/xray/audit/java/mvn.go +++ b/xray/audit/java/mvn.go @@ -13,7 +13,7 @@ import ( xrayUtils "github.com/jfrog/jfrog-client-go/xray/services/utils" ) -func buildMvnDependencyTree(params *DependencyTreeParams) (modules []*xrayUtils.GraphNode, err error) { +func buildMvnDependencyTree(params *DependencyTreeParams) (modules []*xrayUtils.GraphNode, uniqueDeps []string, err error) { buildConfiguration, cleanBuild := createBuildConfiguration("audit-mvn") defer func() { err = errors.Join(err, cleanBuild()) diff --git a/xray/audit/java/mvn_test.go b/xray/audit/java/mvn_test.go index 8ff234572..7cc579788 100644 --- a/xray/audit/java/mvn_test.go +++ b/xray/audit/java/mvn_test.go @@ -14,9 +14,24 @@ func TestMavenTreesMultiModule(t *testing.T) { _, cleanUp := audit.CreateTestWorkspace(t, "maven-example") defer cleanUp() + expectedUniqueDeps := []string{ + GavPackageTypeIdentifier + "javax.mail:mail:1.4", + GavPackageTypeIdentifier + "org.testng:testng:5.9", + GavPackageTypeIdentifier + "javax.servlet:servlet-api:2.5", + GavPackageTypeIdentifier + "org.jfrog.test:multi:3.7-SNAPSHOT", + GavPackageTypeIdentifier + "org.jfrog.test:multi3:3.7-SNAPSHOT", + GavPackageTypeIdentifier + "org.jfrog.test:multi2:3.7-SNAPSHOT", + GavPackageTypeIdentifier + "junit:junit:3.8.1", + GavPackageTypeIdentifier + "org.jfrog.test:multi1:3.7-SNAPSHOT", + GavPackageTypeIdentifier + "commons-io:commons-io:1.4", + GavPackageTypeIdentifier + "org.apache.commons:commons-email:1.1", + GavPackageTypeIdentifier + "javax.activation:activation:1.1", + GavPackageTypeIdentifier + "hsqldb:hsqldb:1.8.0.10", + } // Run getModulesDependencyTrees - modulesDependencyTrees, err := buildMvnDependencyTree(&DependencyTreeParams{IgnoreConfigFile: true}) + modulesDependencyTrees, uniqueDeps, err := buildMvnDependencyTree(&DependencyTreeParams{IgnoreConfigFile: true}) if assert.NoError(t, err) && assert.NotEmpty(t, modulesDependencyTrees) { + assert.ElementsMatch(t, uniqueDeps, expectedUniqueDeps, "First is actual, Second is Expected") // Check root module multi := audit.GetAndAssertNode(t, modulesDependencyTrees, "org.jfrog.test:multi:3.7-SNAPSHOT") if assert.NotNil(t, multi) { @@ -42,8 +57,31 @@ func TestMavenWrapperTrees(t *testing.T) { err := os.Chmod("mvnw", 0700) defer cleanUp() assert.NoError(t, err) - modulesDependencyTrees, err := buildMvnDependencyTree(&DependencyTreeParams{IgnoreConfigFile: true, UseWrapper: true}) + expectedUniqueDeps := []string{ + GavPackageTypeIdentifier + "org.jfrog.test:multi1:3.7-SNAPSHOT", + GavPackageTypeIdentifier + "org.codehaus.plexus:plexus-utils:1.5.1", + GavPackageTypeIdentifier + "org.springframework:spring-beans:2.5.6", + GavPackageTypeIdentifier + "commons-logging:commons-logging:1.1.1", + GavPackageTypeIdentifier + "org.jfrog.test:multi3:3.7-SNAPSHOT", + GavPackageTypeIdentifier + "org.apache.commons:commons-email:1.1", + GavPackageTypeIdentifier + "org.springframework:spring-aop:2.5.6", + GavPackageTypeIdentifier + "org.springframework:spring-core:2.5.6", + GavPackageTypeIdentifier + "org.jfrog.test:multi:3.7-SNAPSHOT", + GavPackageTypeIdentifier + "org.jfrog.test:multi2:3.7-SNAPSHOT", + GavPackageTypeIdentifier + "org.testng:testng:5.9", + GavPackageTypeIdentifier + "hsqldb:hsqldb:1.8.0.10", + GavPackageTypeIdentifier + "junit:junit:3.8.1", + GavPackageTypeIdentifier + "javax.activation:activation:1.1", + GavPackageTypeIdentifier + "javax.mail:mail:1.4", + GavPackageTypeIdentifier + "aopalliance:aopalliance:1.0", + GavPackageTypeIdentifier + "commons-io:commons-io:1.4", + GavPackageTypeIdentifier + "javax.servlet.jsp:jsp-api:2.1", + GavPackageTypeIdentifier + "javax.servlet:servlet-api:2.5", + } + + modulesDependencyTrees, uniqueDeps, err := buildMvnDependencyTree(&DependencyTreeParams{IgnoreConfigFile: true, UseWrapper: true}) if assert.NoError(t, err) && assert.NotEmpty(t, modulesDependencyTrees) { + assert.ElementsMatch(t, uniqueDeps, expectedUniqueDeps, "First is actual, Second is Expected") // Check root module multi := audit.GetAndAssertNode(t, modulesDependencyTrees, "org.jfrog.test:multi:3.7-SNAPSHOT") if assert.NotNil(t, multi) { diff --git a/xray/audit/npm/npm.go b/xray/audit/npm/npm.go index cc1041966..83e926476 100644 --- a/xray/audit/npm/npm.go +++ b/xray/audit/npm/npm.go @@ -15,7 +15,7 @@ const ( ignoreScriptsFlag = "--ignore-scripts" ) -func BuildDependencyTree(npmArgs []string) (dependencyTree []*xrayUtils.GraphNode, err error) { +func BuildDependencyTree(npmArgs []string) (dependencyTrees []*xrayUtils.GraphNode, uniqueDeps []string, err error) { currentDir, err := coreutils.GetWorkingDirectory() if err != nil { return @@ -40,9 +40,9 @@ func BuildDependencyTree(npmArgs []string) (dependencyTree []*xrayUtils.GraphNod for _, dependency := range dependenciesMap { dependenciesList = append(dependenciesList, dependency.Dependency) } - // Parse the dependencies into Xray dependency tree format - dependencyTree = []*xrayUtils.GraphNode{parseNpmDependenciesList(dependenciesList, packageInfo)} + dependencyTree, uniqueDeps := parseNpmDependenciesList(dependenciesList, packageInfo) + dependencyTrees = []*xrayUtils.GraphNode{dependencyTree} return } @@ -55,7 +55,7 @@ func addIgnoreScriptsFlag(npmArgs []string) []string { } // Parse the dependencies into an Xray dependency tree format -func parseNpmDependenciesList(dependencies []buildinfo.Dependency, packageInfo *biutils.PackageInfo) (xrDependencyTree *xrayUtils.GraphNode) { +func parseNpmDependenciesList(dependencies []buildinfo.Dependency, packageInfo *biutils.PackageInfo) (*xrayUtils.GraphNode, []string) { treeMap := make(map[string][]string) for _, dependency := range dependencies { dependencyId := npmPackageTypeIdentifier + dependency.Id diff --git a/xray/audit/npm/npm_test.go b/xray/audit/npm/npm_test.go index afc415de2..a1bfdfcf9 100644 --- a/xray/audit/npm/npm_test.go +++ b/xray/audit/npm/npm_test.go @@ -76,12 +76,11 @@ func TestParseNpmDependenciesList(t *testing.T) { }}, {Id: "npm://next:12.0.10", Nodes: []*xrayUtils.GraphNode{ {Id: "npm://react-dom:18.2.0", Nodes: []*xrayUtils.GraphNode{ - {Id: "npm://react:18.2.0", Nodes: looseEnvifyJsTokens}, - {Id: "npm://loose-envify:1.4.0", Nodes: []*xrayUtils.GraphNode{{Id: "npm://js-tokens:4.0.0"}}}, - {Id: "npm://scheduler:0.23.0", Nodes: looseEnvifyJsTokens}}}, + {Id: "npm://react:18.2.0"}, + {Id: "npm://scheduler:0.23.0"}}}, {Id: "npm://styled-jsx:5.0.0"}, {Id: "npm://@next/swc-darwin-arm64:12.0.10"}, - {Id: "npm://react:18.2.0", Nodes: looseEnvifyJsTokens}, + {Id: "npm://react:18.2.0"}, {Id: "npm://@next/env:12.0.10"}, {Id: "npm://caniuse-lite:1.0.30001486"}, {Id: "npm://postcss:8.4.5", Nodes: []*xrayUtils.GraphNode{ @@ -96,11 +95,16 @@ func TestParseNpmDependenciesList(t *testing.T) { }, } - xrayDependenciesTree := parseNpmDependenciesList(dependencies, packageInfo) + xrayDependenciesTree, uniqueDeps := parseNpmDependenciesList(dependencies, packageInfo) equals := tests.CompareTree(expectedTree, xrayDependenciesTree) if !equals { t.Error("expected:", expectedTree.Nodes, "got:", xrayDependenciesTree.Nodes) } + expectedUniqueDeps := []string{xrayDependenciesTree.Id} + for _, dep := range dependencies { + expectedUniqueDeps = append(expectedUniqueDeps, npmPackageTypeIdentifier+dep.Id) + } + assert.ElementsMatch(t, uniqueDeps, expectedUniqueDeps, "First is actual, Second is Expected") } @@ -111,6 +115,6 @@ func TestIgnoreScripts(t *testing.T) { // The package.json file contain a postinstall script running an "exit 1" command. // Without the "--ignore-scripts" flag, the test will fail. - _, err := BuildDependencyTree([]string{}) + _, _, err := BuildDependencyTree([]string{}) assert.NoError(t, err) } diff --git a/xray/audit/nuget/nuget.go b/xray/audit/nuget/nuget.go index a24024a94..42de0e397 100644 --- a/xray/audit/nuget/nuget.go +++ b/xray/audit/nuget/nuget.go @@ -1,6 +1,7 @@ package nuget import ( + "github.com/jfrog/gofrog/datastructures" "github.com/jfrog/jfrog-client-go/utils/log" xrayUtils "github.com/jfrog/jfrog-client-go/xray/services/utils" "os" @@ -14,7 +15,7 @@ const ( nugetPackageTypeIdentifier = "nuget://" ) -func BuildDependencyTree() (dependencyTree []*xrayUtils.GraphNode, err error) { +func BuildDependencyTree() (dependencyTree []*xrayUtils.GraphNode, uniqueDeps []string, err error) { wd, err := os.Getwd() if err != nil { return @@ -27,11 +28,12 @@ func BuildDependencyTree() (dependencyTree []*xrayUtils.GraphNode, err error) { if err != nil { return } - dependencyTree = parseNugetDependencyTree(buildInfo) + dependencyTree, uniqueDeps = parseNugetDependencyTree(buildInfo) return } -func parseNugetDependencyTree(buildInfo *entities.BuildInfo) (nodes []*xrayUtils.GraphNode) { +func parseNugetDependencyTree(buildInfo *entities.BuildInfo) (nodes []*xrayUtils.GraphNode, allUniqueDeps []string) { + uniqueDepsSet := datastructures.MakeSet[string]() for _, module := range buildInfo.Modules { treeMap := make(map[string][]string) for _, dependency := range module.Dependencies { @@ -43,7 +45,12 @@ func parseNugetDependencyTree(buildInfo *entities.BuildInfo) (nodes []*xrayUtils treeMap[parent] = []string{dependencyId} } } - nodes = append(nodes, audit.BuildXrayDependencyTree(treeMap, nugetPackageTypeIdentifier+module.Id)) + dependencyTree, uniqueDeps := audit.BuildXrayDependencyTree(treeMap, nugetPackageTypeIdentifier+module.Id) + nodes = append(nodes, dependencyTree) + for _, uniqueDep := range uniqueDeps { + uniqueDepsSet.Add(uniqueDep) + } } + allUniqueDeps = uniqueDepsSet.ToSlice() return } diff --git a/xray/audit/nuget/nuget_test.go b/xray/audit/nuget/nuget_test.go index 2584a865e..f8ed17826 100644 --- a/xray/audit/nuget/nuget_test.go +++ b/xray/audit/nuget/nuget_test.go @@ -22,8 +22,20 @@ func TestBuildNugetDependencyTree(t *testing.T) { var dependencies *entities.BuildInfo err = json.Unmarshal(dependenciesJson, &dependencies) assert.NoError(t, err) - xrayDependenciesTree := parseNugetDependencyTree(dependencies) - + expectedUniqueDeps := []string{ + nugetPackageTypeIdentifier + "Microsoft.Net.Http:2.2.29", + nugetPackageTypeIdentifier + "Microsoft.Bcl:1.1.10", + nugetPackageTypeIdentifier + "Microsoft.Bcl.Build:1.0.14", + nugetPackageTypeIdentifier + "Newtonsoft.Json:11.0.2", + nugetPackageTypeIdentifier + "NUnit:3.10.1", + nugetPackageTypeIdentifier + "bootstrap:4.1.1", + nugetPackageTypeIdentifier + "popper.js:1.14.0", + nugetPackageTypeIdentifier + "jQuery:3.0.0", + nugetPackageTypeIdentifier + "MsbuildExample", + nugetPackageTypeIdentifier + "MsbuildLibrary", + } + xrayDependenciesTree, uniqueDeps := parseNugetDependencyTree(dependencies) + assert.ElementsMatch(t, uniqueDeps, expectedUniqueDeps, "First is actual, Second is Expected") expectedTreeJson, err := os.ReadFile("expectedTree.json") assert.NoError(t, err) diff --git a/xray/audit/python/python.go b/xray/audit/python/python.go index 12b3311f6..61eb23f02 100644 --- a/xray/audit/python/python.go +++ b/xray/audit/python/python.go @@ -2,7 +2,9 @@ package python import ( "fmt" + biutils "github.com/jfrog/build-info-go/utils" "github.com/jfrog/build-info-go/utils/pythonutils" + "github.com/jfrog/gofrog/datastructures" "github.com/jfrog/jfrog-cli-core/v2/utils/config" utils "github.com/jfrog/jfrog-cli-core/v2/utils/python" "github.com/jfrog/jfrog-cli-core/v2/xray/audit" @@ -28,25 +30,28 @@ type AuditPython struct { PipRequirementsFile string } -func BuildDependencyTree(auditPython *AuditPython) (dependencyTree []*xrayUtils.GraphNode, err error) { +func BuildDependencyTree(auditPython *AuditPython) (dependencyTree []*xrayUtils.GraphNode, uniqueDeps []string, err error) { dependenciesGraph, directDependenciesList, err := getDependencies(auditPython) if err != nil { return } directDependencies := []*xrayUtils.GraphNode{} + uniqueDepsSet := datastructures.MakeSet[string]() for _, rootDep := range directDependenciesList { directDependency := &xrayUtils.GraphNode{ Id: pythonPackageTypeIdentifier + rootDep, Nodes: []*xrayUtils.GraphNode{}, } - populatePythonDependencyTree(directDependency, dependenciesGraph) + populatePythonDependencyTree(directDependency, dependenciesGraph, uniqueDepsSet) directDependencies = append(directDependencies, directDependency) } root := &xrayUtils.GraphNode{ - Id: pythonPackageTypeIdentifier, + Id: "root", Nodes: directDependencies, } - return []*xrayUtils.GraphNode{root}, nil + dependencyTree = []*xrayUtils.GraphNode{root} + uniqueDeps = uniqueDepsSet.ToSlice() + return } func getDependencies(auditPython *AuditPython) (dependenciesGraph map[string][]string, directDependencies []string, err error) { @@ -78,7 +83,7 @@ func getDependencies(auditPython *AuditPython) (dependenciesGraph map[string][]s } }() - err = fileutils.CopyDir(wd, tempDirPath, true, nil) + err = biutils.CopyDir(wd, tempDirPath, true, nil) if err != nil { return } @@ -271,10 +276,11 @@ func SetPipVirtualEnvPath() (restoreEnv func() error, err error) { return } -func populatePythonDependencyTree(currNode *xrayUtils.GraphNode, dependenciesGraph map[string][]string) { +func populatePythonDependencyTree(currNode *xrayUtils.GraphNode, dependenciesGraph map[string][]string, uniqueDepsSet *datastructures.Set[string]) { if currNode.NodeHasLoop() { return } + uniqueDepsSet.Add(currNode.Id) currDepChildren := dependenciesGraph[strings.TrimPrefix(currNode.Id, pythonPackageTypeIdentifier)] // Recursively create & append all node's dependencies. for _, dependency := range currDepChildren { @@ -284,6 +290,6 @@ func populatePythonDependencyTree(currNode *xrayUtils.GraphNode, dependenciesGra Parent: currNode, } currNode.Nodes = append(currNode.Nodes, childNode) - populatePythonDependencyTree(childNode, dependenciesGraph) + populatePythonDependencyTree(childNode, dependenciesGraph, uniqueDepsSet) } } diff --git a/xray/audit/python/python_test.go b/xray/audit/python/python_test.go index 5817e05d6..3175d299c 100644 --- a/xray/audit/python/python_test.go +++ b/xray/audit/python/python_test.go @@ -14,8 +14,11 @@ func TestBuildPipDependencyListSetuppy(t *testing.T) { _, cleanUp := audit.CreateTestWorkspace(t, filepath.Join("pip-project", "setuppyproject")) defer cleanUp() // Run getModulesDependencyTrees - rootNode, err := BuildDependencyTree(&AuditPython{Tool: pythonutils.Pip}) + rootNode, uniqueDeps, err := BuildDependencyTree(&AuditPython{Tool: pythonutils.Pip}) assert.NoError(t, err) + assert.Contains(t, uniqueDeps, pythonPackageTypeIdentifier+"pexpect:4.8.0") + assert.Contains(t, uniqueDeps, pythonPackageTypeIdentifier+"ptyprocess:0.7.0") + assert.Contains(t, uniqueDeps, pythonPackageTypeIdentifier+"pip-example:1.2.3") assert.Len(t, rootNode, 1) if len(rootNode) > 0 { assert.NotEmpty(t, rootNode[0].Nodes) @@ -35,8 +38,10 @@ func TestPipDependencyListRequirementsFallback(t *testing.T) { _, cleanUp := audit.CreateTestWorkspace(t, filepath.Join("pip-project", "requirementsproject")) defer cleanUp() // No requirements file field specified, expect the command to use the fallback 'pip install -r requirements.txt' command - rootNode, err := BuildDependencyTree(&AuditPython{Tool: pythonutils.Pip}) + rootNode, uniqueDeps, err := BuildDependencyTree(&AuditPython{Tool: pythonutils.Pip}) assert.NoError(t, err) + assert.Contains(t, uniqueDeps, pythonPackageTypeIdentifier+"pexpect:4.7.0") + assert.Contains(t, uniqueDeps, pythonPackageTypeIdentifier+"ptyprocess:0.7.0") assert.Len(t, rootNode, 1) if assert.True(t, len(rootNode[0].Nodes) > 2) { childNode := audit.GetAndAssertNode(t, rootNode[0].Nodes, "pexpect:4.7.0") @@ -52,8 +57,10 @@ func TestBuildPipDependencyListRequirements(t *testing.T) { _, cleanUp := audit.CreateTestWorkspace(t, filepath.Join("pip-project", "requirementsproject")) defer cleanUp() // Run getModulesDependencyTrees - rootNode, err := BuildDependencyTree(&AuditPython{Tool: pythonutils.Pip, PipRequirementsFile: "requirements.txt"}) + rootNode, uniqueDeps, err := BuildDependencyTree(&AuditPython{Tool: pythonutils.Pip, PipRequirementsFile: "requirements.txt"}) assert.NoError(t, err) + assert.Contains(t, uniqueDeps, pythonPackageTypeIdentifier+"pexpect:4.7.0") + assert.Contains(t, uniqueDeps, pythonPackageTypeIdentifier+"ptyprocess:0.7.0") assert.Len(t, rootNode, 1) if len(rootNode) > 0 { assert.NotEmpty(t, rootNode[0].Nodes) @@ -70,11 +77,17 @@ func TestBuildPipenvDependencyList(t *testing.T) { // Create and change directory to test workspace _, cleanUp := audit.CreateTestWorkspace(t, "pipenv-project") defer cleanUp() + expectedPipenvUniqueDeps := []string{ + pythonPackageTypeIdentifier + "toml:0.10.2", + pythonPackageTypeIdentifier + "pexpect:4.8.0", + pythonPackageTypeIdentifier + "ptyprocess:0.7.0", + } // Run getModulesDependencyTrees - rootNode, err := BuildDependencyTree(&AuditPython{Tool: pythonutils.Pipenv}) + rootNode, uniqueDeps, err := BuildDependencyTree(&AuditPython{Tool: pythonutils.Pipenv}) if err != nil { t.Fatal(err) } + assert.ElementsMatch(t, uniqueDeps, expectedPipenvUniqueDeps, "First is actual, Second is Expected") assert.Len(t, rootNode, 1) if len(rootNode) > 0 { assert.NotEmpty(t, rootNode[0].Nodes) @@ -91,11 +104,25 @@ func TestBuildPoetryDependencyList(t *testing.T) { // Create and change directory to test workspace _, cleanUp := audit.CreateTestWorkspace(t, "poetry-project") defer cleanUp() + expectedPoetryUniqueDeps := []string{ + pythonPackageTypeIdentifier + "wcwidth:0.2.5", + pythonPackageTypeIdentifier + "colorama:0.4.6", + pythonPackageTypeIdentifier + "packaging:22.0", + pythonPackageTypeIdentifier + "python:", + pythonPackageTypeIdentifier + "pluggy:0.13.1", + pythonPackageTypeIdentifier + "py:1.11.0", + pythonPackageTypeIdentifier + "atomicwrites:1.4.1", + pythonPackageTypeIdentifier + "attrs:22.1.0", + pythonPackageTypeIdentifier + "more-itertools:9.0.0", + pythonPackageTypeIdentifier + "numpy:1.24.0", + pythonPackageTypeIdentifier + "pytest:5.4.3", + } // Run getModulesDependencyTrees - rootNode, err := BuildDependencyTree(&AuditPython{Tool: pythonutils.Poetry}) + rootNode, uniqueDeps, err := BuildDependencyTree(&AuditPython{Tool: pythonutils.Poetry}) if err != nil { t.Fatal(err) } + assert.ElementsMatch(t, uniqueDeps, expectedPoetryUniqueDeps, "First is actual, Second is Expected") assert.Len(t, rootNode, 1) if len(rootNode) > 0 { assert.NotEmpty(t, rootNode[0].Nodes) diff --git a/xray/audit/yarn/yarn.go b/xray/audit/yarn/yarn.go index 12d987982..41235d597 100644 --- a/xray/audit/yarn/yarn.go +++ b/xray/audit/yarn/yarn.go @@ -13,7 +13,7 @@ const ( npmPackageTypeIdentifier = "npm://" ) -func BuildDependencyTree() (dependencyTree []*xrayUtils.GraphNode, err error) { +func BuildDependencyTree() (dependencyTrees []*xrayUtils.GraphNode, uniqueDeps []string, err error) { currentDir, err := coreutils.GetWorkingDirectory() if err != nil { return @@ -33,12 +33,13 @@ func BuildDependencyTree() (dependencyTree []*xrayUtils.GraphNode, err error) { return } // Parse the dependencies into Xray dependency tree format - dependencyTree = []*xrayUtils.GraphNode{parseYarnDependenciesMap(dependenciesMap, getXrayDependencyId(root))} + dependencyTree, uniqueDeps := parseYarnDependenciesMap(dependenciesMap, getXrayDependencyId(root)) + dependencyTrees = []*xrayUtils.GraphNode{dependencyTree} return } // Parse the dependencies into a Xray dependency tree format -func parseYarnDependenciesMap(dependencies map[string]*biUtils.YarnDependency, rootXrayId string) (xrDependencyTree *xrayUtils.GraphNode) { +func parseYarnDependenciesMap(dependencies map[string]*biUtils.YarnDependency, rootXrayId string) (*xrayUtils.GraphNode, []string) { treeMap := make(map[string][]string) for _, dependency := range dependencies { xrayDepId := getXrayDependencyId(dependency) diff --git a/xray/audit/yarn/yarn_test.go b/xray/audit/yarn/yarn_test.go index c446bb907..bb24f3c0d 100644 --- a/xray/audit/yarn/yarn_test.go +++ b/xray/audit/yarn/yarn_test.go @@ -18,26 +18,33 @@ func TestParseYarnDependenciesList(t *testing.T) { "pack5@npm:5.0.0": {Value: "pack5@npm:5.0.0", Details: biutils.YarnDepDetails{Version: "5.0.0", Dependencies: []biutils.YarnDependencyPointer{{Locator: "pack2@npm:2.0.0"}}}}, } - rootXrayId := "npm://@jfrog/pack3:3.0.0" + rootXrayId := npmPackageTypeIdentifier + "@jfrog/pack3:3.0.0" expectedTree := &xrayUtils.GraphNode{ Id: rootXrayId, Nodes: []*xrayUtils.GraphNode{ - {Id: "npm://pack1:1.0.0", + {Id: npmPackageTypeIdentifier + "pack1:1.0.0", Nodes: []*xrayUtils.GraphNode{ - {Id: "npm://pack4:4.0.0", + {Id: npmPackageTypeIdentifier + "pack4:4.0.0", Nodes: []*xrayUtils.GraphNode{}}, }}, - {Id: "npm://pack2:2.0.0", + {Id: npmPackageTypeIdentifier + "pack2:2.0.0", Nodes: []*xrayUtils.GraphNode{ - {Id: "npm://pack4:4.0.0", + {Id: npmPackageTypeIdentifier + "pack4:4.0.0", Nodes: []*xrayUtils.GraphNode{}}, - {Id: "npm://pack5:5.0.0", + {Id: npmPackageTypeIdentifier + "pack5:5.0.0", Nodes: []*xrayUtils.GraphNode{}}, }}, }, } + expectedUniqueDeps := []string{ + npmPackageTypeIdentifier + "pack1:1.0.0", + npmPackageTypeIdentifier + "pack2:2.0.0", + npmPackageTypeIdentifier + "pack4:4.0.0", + npmPackageTypeIdentifier + "pack5:5.0.0", + npmPackageTypeIdentifier + "@jfrog/pack3:3.0.0", + } - xrayDependenciesTree := parseYarnDependenciesMap(yarnDependencies, rootXrayId) - + xrayDependenciesTree, uniqueDeps := parseYarnDependenciesMap(yarnDependencies, rootXrayId) + assert.ElementsMatch(t, uniqueDeps, expectedUniqueDeps, "First is actual, Second is Expected") assert.True(t, tests.CompareTree(expectedTree, xrayDependenciesTree), "expected:", expectedTree.Nodes, "got:", xrayDependenciesTree.Nodes) } diff --git a/xray/commands/audit/generic/auditmanager.go b/xray/commands/audit/generic/auditmanager.go index 8b535f7a3..7800dc10d 100644 --- a/xray/commands/audit/generic/auditmanager.go +++ b/xray/commands/audit/generic/auditmanager.go @@ -1,6 +1,7 @@ package audit import ( + "encoding/json" "errors" "fmt" "github.com/jfrog/build-info-go/utils/pythonutils" @@ -25,6 +26,7 @@ import ( xrayCmdUtils "github.com/jfrog/jfrog-client-go/xray/services/utils" "golang.org/x/sync/errgroup" "os" + "time" ) type Params struct { @@ -218,7 +220,7 @@ func runScaScanOnWorkingDir(params *Params, results *Results, workingDir, rootDi err = errors.Join(err, fmt.Errorf("failed while building '%s' dependency tree:\n%s\n", tech, techErr.Error())) continue } - if len(flattenTree) == 0 { + if len(flattenTree.Nodes) == 0 { err = errors.Join(err, errors.New("no dependencies were found. Please try to build your project and re-run the audit command")) continue } @@ -235,15 +237,19 @@ func runScaScanOnWorkingDir(params *Params, results *Results, workingDir, rootDi continue } techResults = audit.BuildImpactPathsForScanResponse(techResults, fullDependencyTrees) + var directDependencies []string if tech == coreutils.Pip { - params.AppendDirectDependencies(getDirectDependenciesFromTree(flattenTree)) - + // When building pip dependency tree using pipdeptree, some of the direct dependencies are recognized as transitive and missed by the CA scanner. + // Our solution for this case is to send all dependencies to the CA scanner. + directDependencies = getDirectDependenciesFromTree([]*xrayCmdUtils.GraphNode{flattenTree}) } else { - params.AppendDirectDependencies(getDirectDependenciesFromTree(fullDependencyTrees)) + directDependencies = getDirectDependenciesFromTree(fullDependencyTrees) } + params.AppendDirectDependencies(directDependencies) + results.ExtendedScanResults.XrayResults = append(results.ExtendedScanResults.XrayResults, techResults...) if !results.IsMultipleRootProject { - results.IsMultipleRootProject = len(flattenTree) > 1 + results.IsMultipleRootProject = len(fullDependencyTrees) > 1 } results.ExtendedScanResults.ScannedTechnologies = append(results.ExtendedScanResults.ScannedTechnologies, tech) } @@ -261,7 +267,7 @@ func getDirectDependenciesFromTree(dependencyTrees []*xrayCmdUtils.GraphNode) [] return directDependencies.ToSlice() } -func GetTechDependencyTree(params *xrayutils.GraphBasicParams, tech coreutils.Technology) (flatTree []*xrayCmdUtils.GraphNode, fullDependencyTrees []*xrayCmdUtils.GraphNode, err error) { +func GetTechDependencyTree(params *xrayutils.GraphBasicParams, tech coreutils.Technology) (flatTree *xrayCmdUtils.GraphNode, fullDependencyTrees []*xrayCmdUtils.GraphNode, err error) { if params.Progress() != nil { params.Progress().SetHeadlineMsg(fmt.Sprintf("Calculating %v dependencies", tech.ToFormal())) } @@ -269,46 +275,48 @@ func GetTechDependencyTree(params *xrayutils.GraphBasicParams, tech coreutils.Te if err != nil { return } + var uniqueDeps []string + startTime := time.Now() switch tech { case coreutils.Maven, coreutils.Gradle: - fullDependencyTrees, err = getJavaDependencyTree(params, tech) + fullDependencyTrees, uniqueDeps, err = java.BuildDependencyTree(params, tech) case coreutils.Npm: - fullDependencyTrees, err = npm.BuildDependencyTree(params.Args()) + fullDependencyTrees, uniqueDeps, err = npm.BuildDependencyTree(params.Args()) case coreutils.Yarn: - fullDependencyTrees, err = yarn.BuildDependencyTree() + fullDependencyTrees, uniqueDeps, err = yarn.BuildDependencyTree() case coreutils.Go: - fullDependencyTrees, err = _go.BuildDependencyTree(serverDetails, params.DepsRepo()) + fullDependencyTrees, uniqueDeps, err = _go.BuildDependencyTree(serverDetails, params.DepsRepo()) case coreutils.Pipenv, coreutils.Pip, coreutils.Poetry: - fullDependencyTrees, err = python.BuildDependencyTree(&python.AuditPython{ + fullDependencyTrees, uniqueDeps, err = python.BuildDependencyTree(&python.AuditPython{ Server: serverDetails, Tool: pythonutils.PythonTool(tech), RemotePypiRepo: params.DepsRepo(), PipRequirementsFile: params.PipRequirementsFile()}) case coreutils.Nuget: - fullDependencyTrees, err = nuget.BuildDependencyTree() + fullDependencyTrees, uniqueDeps, err = nuget.BuildDependencyTree() default: err = errorutils.CheckErrorf("%s is currently not supported", string(tech)) } if err != nil { - return nil, nil, err + return } - // Flatten the graph to speed up the ScanGraph request - flatTree, err = services.FlattenGraph(fullDependencyTrees) + log.Debug(fmt.Sprintf("Created '%s' dependency tree with %d nodes. Elapsed time: %.1f seconds.", tech.ToFormal(), len(uniqueDeps), time.Since(startTime).Seconds())) + flatTree, err = createFlatTree(uniqueDeps) return } -func getJavaDependencyTree(params *xrayutils.GraphBasicParams, tech coreutils.Technology) ([]*xrayCmdUtils.GraphNode, error) { - serverDetails, err := params.ServerDetails() - if err != nil { - return nil, err +func createFlatTree(uniqueDeps []string) (*xrayCmdUtils.GraphNode, error) { + if log.GetLogger().GetLogLevel() == log.DEBUG { + // Avoid printing and marshalling if not on DEBUG mode. + jsonList, err := json.Marshal(uniqueDeps) + if errorutils.CheckError(err) != nil { + return nil, err + } + log.Debug("Unique dependencies list:\n" + clientutils.IndentJsonArray(jsonList)) + } + uniqueNodes := []*xrayCmdUtils.GraphNode{} + for _, uniqueDep := range uniqueDeps { + uniqueNodes = append(uniqueNodes, &xrayCmdUtils.GraphNode{Id: uniqueDep}) } - return java.BuildDependencyTree(&java.DependencyTreeParams{ - Tool: tech, - InsecureTls: params.InsecureTls(), - IgnoreConfigFile: params.IgnoreConfigFile(), - ExcludeTestDeps: params.ExcludeTestDependencies(), - UseWrapper: params.UseWrapper(), - Server: serverDetails, - DepsRepo: params.DepsRepo(), - }) + return &xrayCmdUtils.GraphNode{Id: "root", Nodes: uniqueNodes}, nil } diff --git a/xray/commands/curation/audit.go b/xray/commands/curation/audit.go index 9529f59d9..5822f22e7 100644 --- a/xray/commands/curation/audit.go +++ b/xray/commands/curation/audit.go @@ -219,7 +219,7 @@ func (ca *CurationAuditCommand) auditTree(tech coreutils.Technology, results map rootNode := fullDependenciesTree[0] _, projectName, projectScope, projectVersion := getUrlNameAndVersionByTech(tech, rootNode.Id, "", "") if ca.Progress() != nil { - ca.Progress().SetHeadlineMsg(fmt.Sprintf("Fetch curation status for %s graph with %v nodes project name: %s:%s", tech.ToFormal(), len(flattenGraph[0].Nodes)-1, projectName, projectVersion)) + ca.Progress().SetHeadlineMsg(fmt.Sprintf("Fetch curation status for %s graph with %v nodes project name: %s:%s", tech.ToFormal(), len(flattenGraph.Nodes)-1, projectName, projectVersion)) } if projectScope != "" { projectName = projectScope + "/" + projectName @@ -240,7 +240,7 @@ func (ca *CurationAuditCommand) auditTree(tech coreutils.Technology, results map } packagesStatusMap := sync.Map{} // Fetch status for each node from a flatten graph which, has no duplicate nodes. - err = analyzer.fetchNodesStatus(flattenGraph[0], &packagesStatusMap, rootNode.Id) + err = analyzer.fetchNodesStatus(flattenGraph, &packagesStatusMap, rootNode.Id) analyzer.fillGraphRelations(rootNode, &packagesStatusMap, &packagesStatus, "", "", datastructures.MakeSet[string](), true) sort.Slice(packagesStatus, func(i, j int) bool { diff --git a/xray/commands/scan/scan.go b/xray/commands/scan/scan.go index e918da3be..42ad89e42 100644 --- a/xray/commands/scan/scan.go +++ b/xray/commands/scan/scan.go @@ -128,8 +128,8 @@ func (scanCmd *ScanCommand) SetBypassArchiveLimits(bypassArchiveLimits bool) *Sc return scanCmd } -func (scanCmd *ScanCommand) indexFile(filePath string) (*xrayUtils.GraphNode, error) { - var indexerResults xrayUtils.GraphNode +func (scanCmd *ScanCommand) indexFile(filePath string) (*xrayUtils.BinaryGraphNode, error) { + var indexerResults xrayUtils.BinaryGraphNode indexerCmd := exec.Command(scanCmd.indexerPath, indexingCommand, filePath, "--temp-dir", scanCmd.indexerTempDir) if scanCmd.bypassArchiveLimits { indexerCmd.Args = append(indexerCmd.Args, "--bypass-archive-limits") @@ -140,7 +140,8 @@ func (scanCmd *ScanCommand) indexFile(filePath string) (*xrayUtils.GraphNode, er indexerCmd.Stderr = &stderr err := indexerCmd.Run() if err != nil { - if e, ok := err.(*exec.ExitError); ok { + var e *exec.ExitError + if errors.As(err, &e) { if e.ExitCode() == fileNotSupportedExitCode { log.Debug(fmt.Sprintf("File %s is not supported by Xray indexer app.", filePath)) return &indexerResults, nil @@ -158,7 +159,8 @@ func (scanCmd *ScanCommand) indexFile(filePath string) (*xrayUtils.GraphNode, er func (scanCmd *ScanCommand) Run() (err error) { defer func() { if err != nil { - if e, ok := err.(*exec.ExitError); ok { + var e *exec.ExitError + if errors.As(err, &e) { if e.ExitCode() != coreutils.ExitCodeVulnerableBuild.Code { err = errors.New("Scan command failed. " + err.Error()) } @@ -314,7 +316,7 @@ func (scanCmd *ScanCommand) createIndexerHandlerFunc(file *spec.File, indexedFil // which will send the indexed binary to Xray and then will store the received result. taskFunc := func(threadId int) (err error) { params := &services.XrayGraphScanParams{ - Graph: graph, + BinaryGraph: graph, RepoPath: getXrayRepoPathFromTarget(file.Target), Watches: scanCmd.watches, IncludeLicenses: scanCmd.includeLicenses,