Improve Audit and Scan performance for large projects

utils/tests/utils.go

@@ -139,10 +139,10 @@ func ChangeDirAndAssert(t *testing.T, dirPath string) {
 }
 
 // ChangeDirWithCallback changes working directory to the given path and return function that change working directory back to the original path.
-func ChangeDirWithCallback(t *testing.T, wd, dirPath string) func() {
-	ChangeDirAndAssert(t, dirPath)
+func ChangeDirWithCallback(t *testing.T, callbackDir, chdirPath string) func() {
+	ChangeDirAndAssert(t, chdirPath)
 	return func() {
-		ChangeDirAndAssert(t, wd)
+		ChangeDirAndAssert(t, callbackDir)
 	}
 }
 

xray/services/scan.go

@@ -2,10 +2,8 @@ package services
 
 import (
 	"encoding/json"
-	clientutils "github.com/jfrog/jfrog-client-go/utils"
 	"github.com/jfrog/jfrog-client-go/utils/log"
 	xrayUtils "github.com/jfrog/jfrog-client-go/xray/services/utils"
-	"golang.org/x/exp/maps"
 	"net/http"
 	"strings"
 	"time"
@@ -82,7 +80,13 @@ func createScanGraphQueryParams(scanParams XrayGraphScanParams) string {
 func (ss *ScanService) ScanGraph(scanParams XrayGraphScanParams) (string, error) {
 	httpClientsDetails := ss.XrayDetails.CreateHttpClientDetails()
 	utils.SetContentType("application/json", &httpClientsDetails.Headers)
-	requestBody, err := json.Marshal(scanParams.Graph)
+	var err error
+	var requestBody []byte
+	if scanParams.AuditGraph != nil {
+		requestBody, err = json.Marshal(scanParams.AuditGraph)
+	} else {
+		requestBody, err = json.Marshal(scanParams.BinaryGraph)
+	}
 	if err != nil {
 		return "", errorutils.CheckError(err)
 	}
@@ -165,50 +169,12 @@ type XrayGraphScanParams struct {
 	ProjectKey             string
 	Watches                []string
 	ScanType               ScanType
-	Graph                  *xrayUtils.GraphNode
+	AuditGraph             *xrayUtils.GraphNode
+	BinaryGraph            *xrayUtils.BinaryGraphNode
 	IncludeVulnerabilities bool
 	IncludeLicenses        bool
 }
 
-// FlattenGraph creates a map of dependencies from the given graph, and returns a flat graph of dependencies with one level.
-func FlattenGraph(graph []*xrayUtils.GraphNode) ([]*xrayUtils.GraphNode, error) {
-	allDependencies := map[string]*xrayUtils.GraphNode{}
-	for _, node := range graph {
-		populateUniqueDependencies(node, allDependencies)
-	}
-	if log.GetLogger().GetLogLevel() == log.DEBUG {
-		// Print dependencies list only on DEBUG mode.
-		jsonList, err := json.Marshal(maps.Keys(allDependencies))
-		if err != nil {
-			return nil, errorutils.CheckError(err)
-		}
-		log.Debug("Flat dependencies list:\n" + clientutils.IndentJsonArray(jsonList))
-	}
-	return []*xrayUtils.GraphNode{{Id: "root", Nodes: maps.Values(allDependencies)}}, nil
-}
-
-func populateUniqueDependencies(node *xrayUtils.GraphNode, allDependencies map[string]*xrayUtils.GraphNode) {
-	if value, exist := allDependencies[node.Id]; exist &&
-		(len(node.Nodes) == 0 || value.ChildrenExist) {
-		return
-	}
-	allDependencies[node.Id] = &xrayUtils.GraphNode{Id: node.Id}
-	if len(node.Nodes) > 0 {
-		// In some cases node can appear twice, with or without children, this because of the depth limit when creating the graph.
-		// If the node was covered with its children, we mark that, so we won't cover it again.
-		// If its without children, we want to cover it again when it comes with its children.
-		allDependencies[node.Id].ChildrenExist = true
-	}
-	for _, dependency := range node.Nodes {
-		populateUniqueDependencies(dependency, allDependencies)
-	}
-}
-
-type OtherComponentIds struct {
-	Id     string `json:"component_id,omitempty"`
-	Origin int    `json:"origin,omitempty"`
-}
-
 type RequestScanResponse struct {
 	ScanId string `json:"scan_id,omitempty"`
 }

xray/services/scan_test.go

@@ -2,9 +2,6 @@ package services
 
 import (
 	"fmt"
-	"github.com/jfrog/gofrog/datastructures"
-	xrayUtils "github.com/jfrog/jfrog-client-go/xray/services/utils"
-	"github.com/stretchr/testify/assert"
 	"testing"
 )
 
@@ -50,37 +47,3 @@ func TestCreateScanGraphQueryParams(t *testing.T) {
 		})
 	}
 }
-
-func TestFlattenGraph(t *testing.T) {
-	nodeA := &xrayUtils.GraphNode{Id: "A"}
-	nodeB := &xrayUtils.GraphNode{Id: "B"}
-	nodeC := &xrayUtils.GraphNode{Id: "C"}
-	nodeD := &xrayUtils.GraphNode{Id: "D"}
-	nodeE := &xrayUtils.GraphNode{Id: "E"}
-	nodeF := &xrayUtils.GraphNode{Id: "F"}
-	nodeG := &xrayUtils.GraphNode{Id: "G"}
-	nodeGNoChildren := &xrayUtils.GraphNode{Id: "G"}
-	nodeH := &xrayUtils.GraphNode{Id: "H"}
-
-	// Set dependencies
-	nodeA.Nodes = []*xrayUtils.GraphNode{nodeB, nodeC}
-	nodeB.Nodes = []*xrayUtils.GraphNode{nodeC, nodeD}
-	nodeC.Nodes = []*xrayUtils.GraphNode{nodeD}
-	nodeD.Nodes = []*xrayUtils.GraphNode{nodeE, nodeF}
-	nodeF.Nodes = []*xrayUtils.GraphNode{nodeGNoChildren, nodeA, nodeB, nodeC, nodeG}
-	nodeG.Nodes = []*xrayUtils.GraphNode{nodeH}
-
-	// Create graph
-	graph := []*xrayUtils.GraphNode{nodeA, nodeB, nodeC}
-	flatGraph, err := FlattenGraph(graph)
-	assert.NoError(t, err)
-
-	// Check that the graph has been flattened correctly
-	assert.Equal(t, len(flatGraph[0].Nodes), 8)
-	set := datastructures.MakeSet[string]()
-	for _, node := range flatGraph[0].Nodes {
-		assert.Len(t, node.Nodes, 0)
-		assert.False(t, set.Exists(node.Id))
-		set.Add(node.Id)
-	}
-}

xray/services/utils/graph.go

@@ -1,6 +1,7 @@
 package utils
 
-type GraphNode struct {
+// Binary Scan Graph Node
+type BinaryGraphNode struct {
 	// Component Id in the JFrog standard.
 	// For instance, for maven: gav://<groupId>:<artifactId>:<version>
 	// For detailed format examples please see:
@@ -12,28 +13,30 @@ type GraphNode struct {
 	// For root file shall be the file name.
 	// For internal components shall be the internal path. (Relevant only for binary scan).
 	Path string `json:"path,omitempty"`
-	// Download url
-	DownloadUrl string `json:"-"`
 	// List of license names
 	Licenses []string `json:"licenses,omitempty"`
 	// Component properties
 	Properties map[string]string `json:"properties,omitempty"`
 	// List of subcomponents.
-	Nodes []*GraphNode `json:"nodes,omitempty"`
+	Nodes []*BinaryGraphNode `json:"nodes,omitempty"`
 	// Other component IDs field is populated by the Xray indexer to get a better accuracy in '.deb' files.
 	OtherComponentIds []OtherComponentIds `json:"other_component_ids,omitempty"`
-	// Node parent (for internal use)
-	Parent *GraphNode `json:"-"`
-	// Node Can appear in some cases without children. When adding node to flatten graph,
-	// we want to process node again if it was processed without children.
-	ChildrenExist bool `json:"-"`
 }
 
 type OtherComponentIds struct {
 	Id     string `json:"component_id,omitempty"`
 	Origin int    `json:"origin,omitempty"`
 }
 
+// Audit Graph Node
+type GraphNode struct {
+	Id string `json:"component_id,omitempty"`
+	// List of subcomponents.
+	Nodes []*GraphNode `json:"nodes,omitempty"`
+	// Node parent (for internal use)
+	Parent *GraphNode `json:"-"`
+}
+
 func (currNode *GraphNode) NodeHasLoop() bool {
 	parent := currNode.Parent
 	for parent != nil {