From 47d0d428076b0883aa3124ffa47e0f8377569af0 Mon Sep 17 00:00:00 2001 From: Michael Sverdlov Date: Thu, 24 Aug 2023 21:10:06 +0300 Subject: [PATCH 01/12] Improve Audit result table views Signed-off-by: Michael Sverdlov --- utils/tests/utils.go | 6 +++--- xray/services/scan.go | 25 ++++++++++--------------- xray/services/utils/graph.go | 27 ++++++++++++++++++--------- 3 files changed, 31 insertions(+), 27 deletions(-) diff --git a/utils/tests/utils.go b/utils/tests/utils.go index b7b1954d7..495880378 100644 --- a/utils/tests/utils.go +++ b/utils/tests/utils.go @@ -139,10 +139,10 @@ func ChangeDirAndAssert(t *testing.T, dirPath string) { } // ChangeDirWithCallback changes working directory to the given path and return function that change working directory back to the original path. -func ChangeDirWithCallback(t *testing.T, wd, dirPath string) func() { - ChangeDirAndAssert(t, dirPath) +func ChangeDirWithCallback(t *testing.T, callbackDir, chdirPath string) func() { + ChangeDirAndAssert(t, chdirPath) return func() { - ChangeDirAndAssert(t, wd) + ChangeDirAndAssert(t, callbackDir) } } diff --git a/xray/services/scan.go b/xray/services/scan.go index f295f246e..386b945e9 100644 --- a/xray/services/scan.go +++ b/xray/services/scan.go @@ -82,7 +82,13 @@ func createScanGraphQueryParams(scanParams XrayGraphScanParams) string { func (ss *ScanService) ScanGraph(scanParams XrayGraphScanParams) (string, error) { httpClientsDetails := ss.XrayDetails.CreateHttpClientDetails() utils.SetContentType("application/json", &httpClientsDetails.Headers) - requestBody, err := json.Marshal(scanParams.Graph) + var err error + var requestBody []byte + if scanParams.AuditGraph != nil { + requestBody, err = json.Marshal(scanParams.AuditGraph) + } else { + requestBody, err = json.Marshal(scanParams.BinaryGraph) + } if err != nil { return "", errorutils.CheckError(err) } @@ -165,7 +171,8 @@ type XrayGraphScanParams struct { ProjectKey string Watches []string ScanType ScanType - Graph *xrayUtils.GraphNode + AuditGraph *xrayUtils.GraphNode + BinaryGraph *xrayUtils.BinaryGraphNode IncludeVulnerabilities bool IncludeLicenses bool } @@ -188,27 +195,15 @@ func FlattenGraph(graph []*xrayUtils.GraphNode) ([]*xrayUtils.GraphNode, error) } func populateUniqueDependencies(node *xrayUtils.GraphNode, allDependencies map[string]*xrayUtils.GraphNode) { - if value, exist := allDependencies[node.Id]; exist && - (len(node.Nodes) == 0 || value.ChildrenExist) { + if _, exist := allDependencies[node.Id]; exist && len(node.Nodes) == 0 { return } allDependencies[node.Id] = &xrayUtils.GraphNode{Id: node.Id} - if len(node.Nodes) > 0 { - // In some cases node can appear twice, with or without children, this because of the depth limit when creating the graph. - // If the node was covered with its children, we mark that, so we won't cover it again. - // If its without children, we want to cover it again when it comes with its children. - allDependencies[node.Id].ChildrenExist = true - } for _, dependency := range node.Nodes { populateUniqueDependencies(dependency, allDependencies) } } -type OtherComponentIds struct { - Id string `json:"component_id,omitempty"` - Origin int `json:"origin,omitempty"` -} - type RequestScanResponse struct { ScanId string `json:"scan_id,omitempty"` } diff --git a/xray/services/utils/graph.go b/xray/services/utils/graph.go index 706fd26da..3ea1325fc 100644 --- a/xray/services/utils/graph.go +++ b/xray/services/utils/graph.go @@ -1,6 +1,7 @@ package utils -type GraphNode struct { +// Binary Scan Graph Node +type BinaryGraphNode struct { // Component Id in the JFrog standard. // For instance, for maven: gav://:: // For detailed format examples please see: @@ -12,21 +13,14 @@ type GraphNode struct { // For root file shall be the file name. // For internal components shall be the internal path. (Relevant only for binary scan). Path string `json:"path,omitempty"` - // Download url - DownloadUrl string `json:"-"` // List of license names Licenses []string `json:"licenses,omitempty"` // Component properties Properties map[string]string `json:"properties,omitempty"` // List of subcomponents. - Nodes []*GraphNode `json:"nodes,omitempty"` + Nodes []*BinaryGraphNode `json:"nodes,omitempty"` // Other component IDs field is populated by the Xray indexer to get a better accuracy in '.deb' files. OtherComponentIds []OtherComponentIds `json:"other_component_ids,omitempty"` - // Node parent (for internal use) - Parent *GraphNode `json:"-"` - // Node Can appear in some cases without children. When adding node to flatten graph, - // we want to process node again if it was processed without children. - ChildrenExist bool `json:"-"` } type OtherComponentIds struct { @@ -34,6 +28,21 @@ type OtherComponentIds struct { Origin int `json:"origin,omitempty"` } +// Audit Graph Node +type GraphNode struct { + Id string `json:"component_id,omitempty"` + // List of subcomponents. + Nodes []*GraphNode `json:"nodes,omitempty"` + // Node parent (for internal use) + Parent *ParentNode `json:"-"` +} + +type ParentNode struct { + Id string + // Node parent (for internal use) + Parent *ParentNode +} + func (currNode *GraphNode) NodeHasLoop() bool { parent := currNode.Parent for parent != nil { From 8e3df530d1ab84453ef2015eb6e1f95c5714fbb6 Mon Sep 17 00:00:00 2001 From: Michael Sverdlov Date: Sun, 27 Aug 2023 14:55:44 +0300 Subject: [PATCH 02/12] Improve Audit result table views Signed-off-by: Michael Sverdlov --- xray/services/scan.go | 14 ++++++----- xray/services/scan_test.go | 45 +++++++++++++++++++++--------------- xray/services/utils/graph.go | 8 +------ 3 files changed, 35 insertions(+), 32 deletions(-) diff --git a/xray/services/scan.go b/xray/services/scan.go index 386b945e9..67004572b 100644 --- a/xray/services/scan.go +++ b/xray/services/scan.go @@ -181,6 +181,7 @@ type XrayGraphScanParams struct { func FlattenGraph(graph []*xrayUtils.GraphNode) ([]*xrayUtils.GraphNode, error) { allDependencies := map[string]*xrayUtils.GraphNode{} for _, node := range graph { + allDependencies[node.Id] = &xrayUtils.GraphNode{Id: node.Id} populateUniqueDependencies(node, allDependencies) } if log.GetLogger().GetLogLevel() == log.DEBUG { @@ -194,12 +195,13 @@ func FlattenGraph(graph []*xrayUtils.GraphNode) ([]*xrayUtils.GraphNode, error) return []*xrayUtils.GraphNode{{Id: "root", Nodes: maps.Values(allDependencies)}}, nil } -func populateUniqueDependencies(node *xrayUtils.GraphNode, allDependencies map[string]*xrayUtils.GraphNode) { - if _, exist := allDependencies[node.Id]; exist && len(node.Nodes) == 0 { - return - } - allDependencies[node.Id] = &xrayUtils.GraphNode{Id: node.Id} - for _, dependency := range node.Nodes { +func populateUniqueDependencies(currNode *xrayUtils.GraphNode, allDependencies map[string]*xrayUtils.GraphNode) { + for _, dependency := range currNode.Nodes { + dependency.Parent = currNode + if dependency.NodeHasLoop() { + continue + } + allDependencies[dependency.Id] = &xrayUtils.GraphNode{Id: dependency.Id} populateUniqueDependencies(dependency, allDependencies) } } diff --git a/xray/services/scan_test.go b/xray/services/scan_test.go index bc8fb5c47..be2c7de09 100644 --- a/xray/services/scan_test.go +++ b/xray/services/scan_test.go @@ -5,6 +5,7 @@ import ( "github.com/jfrog/gofrog/datastructures" xrayUtils "github.com/jfrog/jfrog-client-go/xray/services/utils" "github.com/stretchr/testify/assert" + "math/rand" "testing" ) @@ -52,27 +53,14 @@ func TestCreateScanGraphQueryParams(t *testing.T) { } func TestFlattenGraph(t *testing.T) { - nodeA := &xrayUtils.GraphNode{Id: "A"} - nodeB := &xrayUtils.GraphNode{Id: "B"} - nodeC := &xrayUtils.GraphNode{Id: "C"} - nodeD := &xrayUtils.GraphNode{Id: "D"} - nodeE := &xrayUtils.GraphNode{Id: "E"} - nodeF := &xrayUtils.GraphNode{Id: "F"} - nodeG := &xrayUtils.GraphNode{Id: "G"} - nodeGNoChildren := &xrayUtils.GraphNode{Id: "G"} - nodeH := &xrayUtils.GraphNode{Id: "H"} - - // Set dependencies - nodeA.Nodes = []*xrayUtils.GraphNode{nodeB, nodeC} - nodeB.Nodes = []*xrayUtils.GraphNode{nodeC, nodeD} - nodeC.Nodes = []*xrayUtils.GraphNode{nodeD} - nodeD.Nodes = []*xrayUtils.GraphNode{nodeE, nodeF} - nodeF.Nodes = []*xrayUtils.GraphNode{nodeGNoChildren, nodeA, nodeB, nodeC, nodeG} - nodeG.Nodes = []*xrayUtils.GraphNode{nodeH} + // Create random trees with the following 8 IDs + depIds := []string{"dep1", "dep2", "dep3", "dep4", "dep5", "dep6", "dep7", "dep8"} + tree1 := generateTreeWithIDs(depIds) + tree2 := generateTreeWithIDs(depIds) + tree3 := generateTreeWithIDs(depIds) // Create graph - graph := []*xrayUtils.GraphNode{nodeA, nodeB, nodeC} - flatGraph, err := FlattenGraph(graph) + flatGraph, err := FlattenGraph([]*xrayUtils.GraphNode{tree1, tree2, tree3}) assert.NoError(t, err) // Check that the graph has been flattened correctly @@ -84,3 +72,22 @@ func TestFlattenGraph(t *testing.T) { set.Add(node.Id) } } + +func generateTreeWithIDs(remainingIDs []string) *xrayUtils.GraphNode { + if len(remainingIDs) == 0 { + return nil + } + + nodeID, remainingIDs := remainingIDs[0], remainingIDs[1:] + node := &xrayUtils.GraphNode{Id: nodeID} + + numChildren := rand.Intn(5) + 1 + for i := 0; i < numChildren; i++ { + child := generateTreeWithIDs(remainingIDs) + if child != nil { + node.Nodes = append(node.Nodes, child) + } + } + + return node +} diff --git a/xray/services/utils/graph.go b/xray/services/utils/graph.go index 3ea1325fc..d976d55a6 100644 --- a/xray/services/utils/graph.go +++ b/xray/services/utils/graph.go @@ -34,13 +34,7 @@ type GraphNode struct { // List of subcomponents. Nodes []*GraphNode `json:"nodes,omitempty"` // Node parent (for internal use) - Parent *ParentNode `json:"-"` -} - -type ParentNode struct { - Id string - // Node parent (for internal use) - Parent *ParentNode + Parent *GraphNode `json:"-"` } func (currNode *GraphNode) NodeHasLoop() bool { From 7d7cd3a0d716262aa57f6dba7edd06e67e7fa348 Mon Sep 17 00:00:00 2001 From: Michael Sverdlov Date: Sun, 27 Aug 2023 17:30:36 +0300 Subject: [PATCH 03/12] Improve Audit result table views Signed-off-by: Michael Sverdlov --- xray/services/scan.go | 4 +++- xray/services/scan_test.go | 23 ++++++++++++++++------- 2 files changed, 19 insertions(+), 8 deletions(-) diff --git a/xray/services/scan.go b/xray/services/scan.go index 67004572b..46d608e4a 100644 --- a/xray/services/scan.go +++ b/xray/services/scan.go @@ -201,7 +201,9 @@ func populateUniqueDependencies(currNode *xrayUtils.GraphNode, allDependencies m if dependency.NodeHasLoop() { continue } - allDependencies[dependency.Id] = &xrayUtils.GraphNode{Id: dependency.Id} + if _, exist := allDependencies[dependency.Id]; !exist { + allDependencies[dependency.Id] = &xrayUtils.GraphNode{Id: dependency.Id} + } populateUniqueDependencies(dependency, allDependencies) } } diff --git a/xray/services/scan_test.go b/xray/services/scan_test.go index be2c7de09..d043b0ee7 100644 --- a/xray/services/scan_test.go +++ b/xray/services/scan_test.go @@ -3,10 +3,12 @@ package services import ( "fmt" "github.com/jfrog/gofrog/datastructures" + "github.com/jfrog/jfrog-client-go/utils/log" xrayUtils "github.com/jfrog/jfrog-client-go/xray/services/utils" "github.com/stretchr/testify/assert" "math/rand" "testing" + "time" ) func TestCreateScanGraphQueryParams(t *testing.T) { @@ -54,17 +56,20 @@ func TestCreateScanGraphQueryParams(t *testing.T) { func TestFlattenGraph(t *testing.T) { // Create random trees with the following 8 IDs + timestamp := time.Now().Unix() + log.Info("Timestamp for test:", timestamp) + seed := rand.New(rand.NewSource(timestamp)) depIds := []string{"dep1", "dep2", "dep3", "dep4", "dep5", "dep6", "dep7", "dep8"} - tree1 := generateTreeWithIDs(depIds) - tree2 := generateTreeWithIDs(depIds) - tree3 := generateTreeWithIDs(depIds) + tree1 := generateTreeWithIDs(depIds, seed) + tree2 := generateTreeWithIDs(depIds, seed) + tree3 := generateTreeWithIDs(depIds, seed) // Create graph flatGraph, err := FlattenGraph([]*xrayUtils.GraphNode{tree1, tree2, tree3}) assert.NoError(t, err) // Check that the graph has been flattened correctly - assert.Equal(t, len(flatGraph[0].Nodes), 8) + assert.Len(t, flatGraph[0].Nodes, 8) set := datastructures.MakeSet[string]() for _, node := range flatGraph[0].Nodes { assert.Len(t, node.Nodes, 0) @@ -73,17 +78,21 @@ func TestFlattenGraph(t *testing.T) { } } -func generateTreeWithIDs(remainingIDs []string) *xrayUtils.GraphNode { +func generateTreeWithIDs(remainingIDs []string, seed *rand.Rand) *xrayUtils.GraphNode { if len(remainingIDs) == 0 { return nil } + // Shuffle IDs + seed.Shuffle(len(remainingIDs), func(i, j int) { + remainingIDs[i], remainingIDs[j] = remainingIDs[j], remainingIDs[i] + }) nodeID, remainingIDs := remainingIDs[0], remainingIDs[1:] node := &xrayUtils.GraphNode{Id: nodeID} - numChildren := rand.Intn(5) + 1 + numChildren := seed.Intn(5) + 1 for i := 0; i < numChildren; i++ { - child := generateTreeWithIDs(remainingIDs) + child := generateTreeWithIDs(remainingIDs, seed) if child != nil { node.Nodes = append(node.Nodes, child) } From 08bb34cba08699ad6357d92d06c06badb2f62184 Mon Sep 17 00:00:00 2001 From: Michael Sverdlov Date: Sun, 27 Aug 2023 17:46:52 +0300 Subject: [PATCH 04/12] Improve Audit result table views Signed-off-by: Michael Sverdlov --- xray/services/scan.go | 4 ++-- xray/services/scan_test.go | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/xray/services/scan.go b/xray/services/scan.go index 46d608e4a..bbb00a24a 100644 --- a/xray/services/scan.go +++ b/xray/services/scan.go @@ -178,7 +178,7 @@ type XrayGraphScanParams struct { } // FlattenGraph creates a map of dependencies from the given graph, and returns a flat graph of dependencies with one level. -func FlattenGraph(graph []*xrayUtils.GraphNode) ([]*xrayUtils.GraphNode, error) { +func FlattenGraph(graph []*xrayUtils.GraphNode) (*xrayUtils.GraphNode, error) { allDependencies := map[string]*xrayUtils.GraphNode{} for _, node := range graph { allDependencies[node.Id] = &xrayUtils.GraphNode{Id: node.Id} @@ -192,7 +192,7 @@ func FlattenGraph(graph []*xrayUtils.GraphNode) ([]*xrayUtils.GraphNode, error) } log.Debug("Flat dependencies list:\n" + clientutils.IndentJsonArray(jsonList)) } - return []*xrayUtils.GraphNode{{Id: "root", Nodes: maps.Values(allDependencies)}}, nil + return &xrayUtils.GraphNode{Id: "root", Nodes: maps.Values(allDependencies)}, nil } func populateUniqueDependencies(currNode *xrayUtils.GraphNode, allDependencies map[string]*xrayUtils.GraphNode) { diff --git a/xray/services/scan_test.go b/xray/services/scan_test.go index d043b0ee7..72d8c9bc3 100644 --- a/xray/services/scan_test.go +++ b/xray/services/scan_test.go @@ -69,9 +69,9 @@ func TestFlattenGraph(t *testing.T) { assert.NoError(t, err) // Check that the graph has been flattened correctly - assert.Len(t, flatGraph[0].Nodes, 8) + assert.Len(t, flatGraph.Nodes, 8) set := datastructures.MakeSet[string]() - for _, node := range flatGraph[0].Nodes { + for _, node := range flatGraph.Nodes { assert.Len(t, node.Nodes, 0) assert.False(t, set.Exists(node.Id)) set.Add(node.Id) From ccda48c7092639d7656772cc636d0f026c9f9163 Mon Sep 17 00:00:00 2001 From: Michael Sverdlov Date: Sun, 27 Aug 2023 17:49:38 +0300 Subject: [PATCH 05/12] Improve Audit result table views Signed-off-by: Michael Sverdlov --- xray/services/scan_test.go | 1 + 1 file changed, 1 insertion(+) diff --git a/xray/services/scan_test.go b/xray/services/scan_test.go index 72d8c9bc3..184073147 100644 --- a/xray/services/scan_test.go +++ b/xray/services/scan_test.go @@ -58,6 +58,7 @@ func TestFlattenGraph(t *testing.T) { // Create random trees with the following 8 IDs timestamp := time.Now().Unix() log.Info("Timestamp for test:", timestamp) + //#nosec G404 seed := rand.New(rand.NewSource(timestamp)) depIds := []string{"dep1", "dep2", "dep3", "dep4", "dep5", "dep6", "dep7", "dep8"} tree1 := generateTreeWithIDs(depIds, seed) From 3b31491972e0d8dc41395dc4de24bb31848ce60a Mon Sep 17 00:00:00 2001 From: Michael Sverdlov Date: Mon, 28 Aug 2023 18:44:06 +0300 Subject: [PATCH 06/12] Improve Audit result table views Signed-off-by: Michael Sverdlov --- xray/services/scan.go | 33 ----------------------- xray/services/scan_test.go | 54 -------------------------------------- 2 files changed, 87 deletions(-) diff --git a/xray/services/scan.go b/xray/services/scan.go index bbb00a24a..e3ae67f20 100644 --- a/xray/services/scan.go +++ b/xray/services/scan.go @@ -2,10 +2,8 @@ package services import ( "encoding/json" - clientutils "github.com/jfrog/jfrog-client-go/utils" "github.com/jfrog/jfrog-client-go/utils/log" xrayUtils "github.com/jfrog/jfrog-client-go/xray/services/utils" - "golang.org/x/exp/maps" "net/http" "strings" "time" @@ -177,37 +175,6 @@ type XrayGraphScanParams struct { IncludeLicenses bool } -// FlattenGraph creates a map of dependencies from the given graph, and returns a flat graph of dependencies with one level. -func FlattenGraph(graph []*xrayUtils.GraphNode) (*xrayUtils.GraphNode, error) { - allDependencies := map[string]*xrayUtils.GraphNode{} - for _, node := range graph { - allDependencies[node.Id] = &xrayUtils.GraphNode{Id: node.Id} - populateUniqueDependencies(node, allDependencies) - } - if log.GetLogger().GetLogLevel() == log.DEBUG { - // Print dependencies list only on DEBUG mode. - jsonList, err := json.Marshal(maps.Keys(allDependencies)) - if err != nil { - return nil, errorutils.CheckError(err) - } - log.Debug("Flat dependencies list:\n" + clientutils.IndentJsonArray(jsonList)) - } - return &xrayUtils.GraphNode{Id: "root", Nodes: maps.Values(allDependencies)}, nil -} - -func populateUniqueDependencies(currNode *xrayUtils.GraphNode, allDependencies map[string]*xrayUtils.GraphNode) { - for _, dependency := range currNode.Nodes { - dependency.Parent = currNode - if dependency.NodeHasLoop() { - continue - } - if _, exist := allDependencies[dependency.Id]; !exist { - allDependencies[dependency.Id] = &xrayUtils.GraphNode{Id: dependency.Id} - } - populateUniqueDependencies(dependency, allDependencies) - } -} - type RequestScanResponse struct { ScanId string `json:"scan_id,omitempty"` } diff --git a/xray/services/scan_test.go b/xray/services/scan_test.go index 184073147..1eafe2a21 100644 --- a/xray/services/scan_test.go +++ b/xray/services/scan_test.go @@ -2,13 +2,7 @@ package services import ( "fmt" - "github.com/jfrog/gofrog/datastructures" - "github.com/jfrog/jfrog-client-go/utils/log" - xrayUtils "github.com/jfrog/jfrog-client-go/xray/services/utils" - "github.com/stretchr/testify/assert" - "math/rand" "testing" - "time" ) func TestCreateScanGraphQueryParams(t *testing.T) { @@ -53,51 +47,3 @@ func TestCreateScanGraphQueryParams(t *testing.T) { }) } } - -func TestFlattenGraph(t *testing.T) { - // Create random trees with the following 8 IDs - timestamp := time.Now().Unix() - log.Info("Timestamp for test:", timestamp) - //#nosec G404 - seed := rand.New(rand.NewSource(timestamp)) - depIds := []string{"dep1", "dep2", "dep3", "dep4", "dep5", "dep6", "dep7", "dep8"} - tree1 := generateTreeWithIDs(depIds, seed) - tree2 := generateTreeWithIDs(depIds, seed) - tree3 := generateTreeWithIDs(depIds, seed) - - // Create graph - flatGraph, err := FlattenGraph([]*xrayUtils.GraphNode{tree1, tree2, tree3}) - assert.NoError(t, err) - - // Check that the graph has been flattened correctly - assert.Len(t, flatGraph.Nodes, 8) - set := datastructures.MakeSet[string]() - for _, node := range flatGraph.Nodes { - assert.Len(t, node.Nodes, 0) - assert.False(t, set.Exists(node.Id)) - set.Add(node.Id) - } -} - -func generateTreeWithIDs(remainingIDs []string, seed *rand.Rand) *xrayUtils.GraphNode { - if len(remainingIDs) == 0 { - return nil - } - // Shuffle IDs - seed.Shuffle(len(remainingIDs), func(i, j int) { - remainingIDs[i], remainingIDs[j] = remainingIDs[j], remainingIDs[i] - }) - - nodeID, remainingIDs := remainingIDs[0], remainingIDs[1:] - node := &xrayUtils.GraphNode{Id: nodeID} - - numChildren := seed.Intn(5) + 1 - for i := 0; i < numChildren; i++ { - child := generateTreeWithIDs(remainingIDs, seed) - if child != nil { - node.Nodes = append(node.Nodes, child) - } - } - - return node -} From 29f031f113b399bfafebc963c68fee3be610403a Mon Sep 17 00:00:00 2001 From: Michael Sverdlov Date: Tue, 29 Aug 2023 16:03:34 +0300 Subject: [PATCH 07/12] Improve Audit result table views Signed-off-by: Michael Sverdlov --- utils/tests/utils.go | 6 +++--- xray/services/scan.go | 16 +++++++++------- 2 files changed, 12 insertions(+), 10 deletions(-) diff --git a/utils/tests/utils.go b/utils/tests/utils.go index 495880378..3bfef9a3a 100644 --- a/utils/tests/utils.go +++ b/utils/tests/utils.go @@ -139,10 +139,10 @@ func ChangeDirAndAssert(t *testing.T, dirPath string) { } // ChangeDirWithCallback changes working directory to the given path and return function that change working directory back to the original path. -func ChangeDirWithCallback(t *testing.T, callbackDir, chdirPath string) func() { - ChangeDirAndAssert(t, chdirPath) +func ChangeDirWithCallback(t *testing.T, originWd, destinationWd string) func() { + ChangeDirAndAssert(t, destinationWd) return func() { - ChangeDirAndAssert(t, callbackDir) + ChangeDirAndAssert(t, originWd) } } diff --git a/xray/services/scan.go b/xray/services/scan.go index e3ae67f20..13415d9e9 100644 --- a/xray/services/scan.go +++ b/xray/services/scan.go @@ -82,8 +82,8 @@ func (ss *ScanService) ScanGraph(scanParams XrayGraphScanParams) (string, error) utils.SetContentType("application/json", &httpClientsDetails.Headers) var err error var requestBody []byte - if scanParams.AuditGraph != nil { - requestBody, err = json.Marshal(scanParams.AuditGraph) + if scanParams.DependenciesGraph != nil { + requestBody, err = json.Marshal(scanParams.DependenciesGraph) } else { requestBody, err = json.Marshal(scanParams.BinaryGraph) } @@ -165,11 +165,13 @@ func (ss *ScanService) GetScanGraphResults(scanId string, includeVulnerabilities type XrayGraphScanParams struct { // A path in Artifactory that this Artifact is intended to be deployed to. // This will provide a way to extract the watches that should be applied on this graph - RepoPath string - ProjectKey string - Watches []string - ScanType ScanType - AuditGraph *xrayUtils.GraphNode + RepoPath string + ProjectKey string + Watches []string + ScanType ScanType + // Dependencies Tree + DependenciesGraph *xrayUtils.GraphNode + // Binary tree received from indexer-app BinaryGraph *xrayUtils.BinaryGraphNode IncludeVulnerabilities bool IncludeLicenses bool From 46fc8fefe589df41f2f46fc6830746e55b90e7b2 Mon Sep 17 00:00:00 2001 From: Michael Sverdlov Date: Tue, 29 Aug 2023 16:19:06 +0300 Subject: [PATCH 08/12] Improve Audit result table views Signed-off-by: Michael Sverdlov --- .github/workflows/frogbot-scan-and-fix.yml | 40 ------ .github/workflows/frogbot-scan-pr.yml | 34 ----- .../workflows/frogbot-scan-pull-request.yml | 127 ++++++++++++++++++ .github/workflows/frogbot-scan-repository.yml | 125 +++++++++++++++++ 4 files changed, 252 insertions(+), 74 deletions(-) delete mode 100644 .github/workflows/frogbot-scan-and-fix.yml delete mode 100644 .github/workflows/frogbot-scan-pr.yml create mode 100644 .github/workflows/frogbot-scan-pull-request.yml create mode 100644 .github/workflows/frogbot-scan-repository.yml diff --git a/.github/workflows/frogbot-scan-and-fix.yml b/.github/workflows/frogbot-scan-and-fix.yml deleted file mode 100644 index aa34c12cc..000000000 --- a/.github/workflows/frogbot-scan-and-fix.yml +++ /dev/null @@ -1,40 +0,0 @@ -name: "Frogbot Scan and Fix" -on: - schedule: - # The repository will be scanned once a day at 00:00 GMT. - - cron: "0 0 * * *" -permissions: - contents: write - pull-requests: write - security-events: write -jobs: - create-fix-pull-requests: - runs-on: ubuntu-latest - strategy: - matrix: - # The repository scanning will be triggered periodically on the following branches. - branch: [ "dev" ] - steps: - - uses: actions/checkout@v3 - with: - ref: ${{ matrix.branch }} - - # Install prerequisites - - name: Setup Go - uses: actions/setup-go@v3 - with: - go-version: 1.20.x - - - uses: jfrog/frogbot@v2 - env: - # [Mandatory] - # JFrog platform URL - JF_URL: ${{ secrets.FROGBOT_URL }} - - # [Mandatory if JF_USER and JF_PASSWORD are not provided] - # JFrog access token with 'read' permissions on Xray service - JF_ACCESS_TOKEN: ${{ secrets.FROGBOT_ACCESS_TOKEN }} - - # [Mandatory] - # The GitHub token automatically generated for the job - JF_GIT_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/frogbot-scan-pr.yml b/.github/workflows/frogbot-scan-pr.yml deleted file mode 100644 index 8b61e89d1..000000000 --- a/.github/workflows/frogbot-scan-pr.yml +++ /dev/null @@ -1,34 +0,0 @@ -name: "Frogbot Scan PR" -on: - pull_request_target: - types: [ opened, synchronize ] -jobs: - frogbot: - runs-on: ubuntu-latest - # A pull request needs to be approved, before Frogbot scans it. Any GitHub user who is associated with the - # "frogbot" GitHub environment can approve the pull request to be scanned. - environment: frogbot - steps: - - uses: actions/checkout@v3 - with: - ref: ${{ github.event.pull_request.head.sha }} - - # Install prerequisites - - name: Setup Go - uses: actions/setup-go@v3 - with: - go-version: 1.20.x - - - uses: jfrog/frogbot@v2 - env: - # [Mandatory] - # JFrog platform URL - JF_URL: ${{ secrets.FROGBOT_URL }} - - # [Mandatory if JF_USER and JF_PASSWORD are not provided] - # JFrog access token with 'read' permissions on Xray service - JF_ACCESS_TOKEN: ${{ secrets.FROGBOT_ACCESS_TOKEN }} - - # [Mandatory] - # The GitHub token automatically generated for the job - JF_GIT_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/frogbot-scan-pull-request.yml b/.github/workflows/frogbot-scan-pull-request.yml new file mode 100644 index 000000000..ff10f38bc --- /dev/null +++ b/.github/workflows/frogbot-scan-pull-request.yml @@ -0,0 +1,127 @@ +name: "Frogbot Scan Pull Request" +on: + pull_request_target: + types: [ opened, synchronize ] +permissions: + pull-requests: write + contents: read +jobs: + scan-pull-request: + runs-on: ubuntu-latest + # A pull request needs to be approved before Frogbot scans it. Any GitHub user who is associated with the + # "frogbot" GitHub environment can approve the pull request to be scanned. + environment: frogbot + steps: + - uses: jfrog/frogbot@v2 + env: + JFROG_CLI_LOG_LEVEL: "DEBUG" + # [Mandatory] + # JFrog platform URL (This functionality requires version 3.29.0 or above of Xray) + JF_URL: ${{ secrets.FROGBOT_URL }} + + # [Mandatory if JF_USER and JF_PASSWORD are not provided] + # JFrog access token with 'read' permissions on Xray service + JF_ACCESS_TOKEN: ${{ secrets.FROGBOT_ACCESS_TOKEN }} + + # [Mandatory if JF_ACCESS_TOKEN is not provided] + # JFrog username with 'read' permissions for Xray. Must be provided with JF_PASSWORD + # JF_USER: ${{ secrets.JF_USER }} + + # [Mandatory if JF_ACCESS_TOKEN is not provided] + # JFrog password. Must be provided with JF_USER + # JF_PASSWORD: ${{ secrets.JF_PASSWORD }} + + # [Mandatory] + # The GitHub token is automatically generated for the job + JF_GIT_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + # [Optional, default: https://api.github.com] + # API endpoint to GitHub + # JF_GIT_API_ENDPOINT: https://github.example.com + + # [Optional] + # By default, the Frogbot workflows download the Frogbot executable as well as other tools + # needed from https://releases.jfrog.io + # If the machine that runs Frogbot has no access to the internet, follow these steps to allow the + # executable to be downloaded from an Artifactory instance, which the machine has access to: + # + # 1. Login to the Artifactory UI, with a user who has admin credentials. + # 2. Create a Remote Repository with the following properties set. + # Under the 'Basic' tab: + # Package Type: Generic + # URL: https://releases.jfrog.io + # Under the 'Advanced' tab: + # Uncheck the 'Store Artifacts Locally' option + # 3. Set the value of the 'JF_RELEASES_REPO' variable with the Repository Key you created. + # JF_RELEASES_REPO: "" + + # [Optional] + # Configure the SMTP server to enable Frogbot to send emails with detected secrets in pull request scans. + # SMTP server URL including should the relevant port: (Example: smtp.server.com:8080) + JF_SMTP_SERVER: ${{ secrets.JF_SMTP_SERVER }} + + # [Mandatory if JF_SMTP_SERVER is set] + # The username required for authenticating with the SMTP server. + JF_SMTP_USER: ${{ secrets.JF_SMTP_USER }} + + # [Mandatory if JF_SMTP_SERVER is set] + # The password associated with the username required for authentication with the SMTP server. + JF_SMTP_PASSWORD: ${{ secrets.JF_SMTP_PASSWORD }} + + ########################################################################## + ## If your project uses a 'frogbot-config.yml' file, you can define ## + ## the following variables inside the file, instead of here. ## + ########################################################################## + + # [Mandatory if the two conditions below are met] + # 1. The project uses yarn 2, NuGet or .NET Core to download its dependencies + # 2. The `installCommand` variable isn't set in your frogbot-config.yml file. + # + # The command that installs the project dependencies (e.g "nuget restore") + # JF_INSTALL_DEPS_CMD: "" + + # [Optional, default: "."] + # Relative path to the root of the project in the Git repository + # JF_WORKING_DIR: path/to/project/dir + + # [Optional] + # Xray Watches. Learn more about them here: https://www.jfrog.com/confluence/display/JFROG/Configuring+Xray+Watches + # JF_WATCHES: ,... + + # [Optional] + # JFrog project. Learn more about it here: https://www.jfrog.com/confluence/display/JFROG/Projects + # JF_PROJECT: + + # [Optional, default: "FALSE"] + # Displays all existing vulnerabilities, including the ones that were added by the pull request. + # JF_INCLUDE_ALL_VULNERABILITIES: "TRUE" + + # [Optional, default: "TRUE"] + # Fails the Frogbot task if any security issue is found. + # JF_FAIL: "FALSE" + + # [Optional] + # Frogbot will download the project dependencies if they're not cached locally. To download the + # dependencies from a virtual repository in Artifactory, set the name of the repository. There's no + # need to set this value, if it is set in the frogbot-config.yml file. + # JF_DEPS_REPO: "" + + # [Optional, Default: "FALSE"] + # If TRUE, Frogbot creates a single pull request with all the fixes. + # If false, Frogbot creates a separate pull request for each fix. + # JF_GIT_AGGREGATE_FIXES: "FALSE" + + # [Optional, Default: "FALSE"] + # Handle vulnerabilities with fix versions only + # JF_FIXABLE_ONLY: "TRUE" + + # [Optional] + # Set the minimum severity for vulnerabilities that should be fixed and commented on in pull requests + # The following values are accepted: Low, Medium, High or Critical + # JF_MIN_SEVERITY: "" + + # [Optional] + # List of comma separated email addresses to receive email notifications about secrets + # detected during pull request scanning. The notification is also sent to the email set + # in the committer git profile regardless of whether this variable is set or not. + # JF_EMAIL_RECEIVERS: "" \ No newline at end of file diff --git a/.github/workflows/frogbot-scan-repository.yml b/.github/workflows/frogbot-scan-repository.yml new file mode 100644 index 000000000..01b568f67 --- /dev/null +++ b/.github/workflows/frogbot-scan-repository.yml @@ -0,0 +1,125 @@ +name: "Frogbot Scan Repository" +on: + workflow_dispatch: + schedule: + # The repository will be scanned once a day at 00:00 GMT. + - cron: "0 0 * * *" +permissions: + contents: write + pull-requests: write + security-events: write +jobs: + scan-repository: + runs-on: ubuntu-latest + strategy: + matrix: + # The repository scanning will be triggered periodically on the following branches. + branch: [ "dev" ] + steps: + - uses: jfrog/frogbot@v2 + env: + JFROG_CLI_LOG_LEVEL: "DEBUG" + # [Mandatory] + # JFrog platform URL (This functionality requires version 3.29.0 or above of Xray) + JF_URL: ${{ secrets.FROGBOT_URL }} + + # [Mandatory if JF_USER and JF_PASSWORD are not provided] + # JFrog access token with 'read' permissions on Xray service + JF_ACCESS_TOKEN: ${{ secrets.FROGBOT_ACCESS_TOKEN }} + + # [Mandatory if JF_ACCESS_TOKEN is not provided] + # JFrog username with 'read' permissions for Xray. Must be provided with JF_PASSWORD + # JF_USER: ${{ secrets.JF_USER }} + + # [Mandatory if JF_ACCESS_TOKEN is not provided] + # JFrog password. Must be provided with JF_USER + # JF_PASSWORD: ${{ secrets.JF_PASSWORD }} + + # [Mandatory] + # The GitHub token is automatically generated for the job + JF_GIT_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + # [Mandatory] + # The name of the branch on which Frogbot will perform the scan + JF_GIT_BASE_BRANCH: ${{ matrix.branch }} + + # [Optional, default: https://api.github.com] + # API endpoint to GitHub + # JF_GIT_API_ENDPOINT: https://github.example.com + + # [Optional] + # By default, the Frogbot workflows download the Frogbot executable as well as other tools + # needed from https://releases.jfrog.io + # If the machine that runs Frogbot has no access to the internet, follow these steps to allow the + # executable to be downloaded from an Artifactory instance, which the machine has access to: + # + # 1. Login to the Artifactory UI, with a user who has admin credentials. + # 2. Create a Remote Repository with the following properties set. + # Under the 'Basic' tab: + # Package Type: Generic + # URL: https://releases.jfrog.io + # Under the 'Advanced' tab: + # Uncheck the 'Store Artifacts Locally' option + # 3. Set the value of the 'JF_RELEASES_REPO' variable with the Repository Key you created. + # JF_RELEASES_REPO: "" + + ########################################################################## + ## If your project uses a 'frogbot-config.yml' file, you can define ## + ## the following variables inside the file, instead of here. ## + ########################################################################## + + # [Optional, default: "."] + # Relative path to the root of the project in the Git repository + # JF_WORKING_DIR: path/to/project/dir + + # [Optional] + # Xray Watches. Learn more about them here: https://www.jfrog.com/confluence/display/JFROG/Configuring+Xray+Watches + # JF_WATCHES: ,... + + # [Optional] + # JFrog project. Learn more about it here: https://www.jfrog.com/confluence/display/JFROG/Projects + # JF_PROJECT: + + # [Optional, default: "TRUE"] + # Fails the Frogbot task if any security issue is found. + # JF_FAIL: "FALSE" + + # [Optional] + # Frogbot will download the project dependencies, if they're not cached locally. To download the + # dependencies from a virtual repository in Artifactory, set the name of the repository. There's no + # need to set this value, if it is set in the frogbot-config.yml file. + # JF_DEPS_REPO: "" + + # [Optional] + # Template for the branch name generated by Frogbot when creating pull requests with fixes. + # The template must include ${BRANCH_NAME_HASH}, to ensure that the generated branch name is unique. + # The template can optionally include the ${IMPACTED_PACKAGE} and ${FIX_VERSION} variables. + # JF_BRANCH_NAME_TEMPLATE: "frogbot-${IMPACTED_PACKAGE}-${BRANCH_NAME_HASH}" + + # [Optional] + # Template for the commit message generated by Frogbot when creating pull requests with fixes + # The template can optionally include the ${IMPACTED_PACKAGE} and ${FIX_VERSION} variables. + # JF_COMMIT_MESSAGE_TEMPLATE: "Upgrade ${IMPACTED_PACKAGE} to ${FIX_VERSION}" + + # [Optional] + # Template for the pull request title generated by Frogbot when creating pull requests with fixes. + # The template can optionally include the ${IMPACTED_PACKAGE} and ${FIX_VERSION} variables. + # JF_PULL_REQUEST_TITLE_TEMPLATE: "[🐸 Frogbot] Upgrade ${IMPACTED_PACKAGE} to ${FIX_VERSION}" + + # [Optional, Default: "FALSE"] + # If TRUE, Frogbot creates a single pull request with all the fixes. + # If FALSE, Frogbot creates a separate pull request for each fix. + # JF_GIT_AGGREGATE_FIXES: "FALSE" + + # [Optional, Default: "FALSE"] + # Handle vulnerabilities with fix versions only + # JF_FIXABLE_ONLY: "TRUE" + + # [Optional] + # Set the minimum severity for vulnerabilities that should be fixed and commented on in pull requests + # The following values are accepted: Low, Medium, High or Critical + # JF_MIN_SEVERITY: "" + + # [Optional, Default: eco-system+frogbot@jfrog.com] + # Set the email of the commit author + # JF_GIT_EMAIL_AUTHOR: "" \ No newline at end of file From 817c5f8cf7a5b90044e83af1d7411c0e73e58e8d Mon Sep 17 00:00:00 2001 From: Michael Sverdlov Date: Wed, 30 Aug 2023 21:48:06 +0300 Subject: [PATCH 09/12] Improve Audit result table views Signed-off-by: Michael Sverdlov --- .../workflows/frogbot-scan-pull-request.yml | 22 ++++++------------- 1 file changed, 7 insertions(+), 15 deletions(-) diff --git a/.github/workflows/frogbot-scan-pull-request.yml b/.github/workflows/frogbot-scan-pull-request.yml index ff10f38bc..998c8c91f 100644 --- a/.github/workflows/frogbot-scan-pull-request.yml +++ b/.github/workflows/frogbot-scan-pull-request.yml @@ -23,14 +23,6 @@ jobs: # JFrog access token with 'read' permissions on Xray service JF_ACCESS_TOKEN: ${{ secrets.FROGBOT_ACCESS_TOKEN }} - # [Mandatory if JF_ACCESS_TOKEN is not provided] - # JFrog username with 'read' permissions for Xray. Must be provided with JF_PASSWORD - # JF_USER: ${{ secrets.JF_USER }} - - # [Mandatory if JF_ACCESS_TOKEN is not provided] - # JFrog password. Must be provided with JF_USER - # JF_PASSWORD: ${{ secrets.JF_PASSWORD }} - # [Mandatory] # The GitHub token is automatically generated for the job JF_GIT_TOKEN: ${{ secrets.GITHUB_TOKEN }} @@ -68,6 +60,12 @@ jobs: # The password associated with the username required for authentication with the SMTP server. JF_SMTP_PASSWORD: ${{ secrets.JF_SMTP_PASSWORD }} + # [Optional] + # List of comma separated email addresses to receive email notifications about secrets + # detected during pull request scanning. The notification is also sent to the email set + # in the committer git profile regardless of whether this variable is set or not. + JF_EMAIL_RECEIVERS: "eco-system@jfrog.com" + ########################################################################## ## If your project uses a 'frogbot-config.yml' file, you can define ## ## the following variables inside the file, instead of here. ## @@ -118,10 +116,4 @@ jobs: # [Optional] # Set the minimum severity for vulnerabilities that should be fixed and commented on in pull requests # The following values are accepted: Low, Medium, High or Critical - # JF_MIN_SEVERITY: "" - - # [Optional] - # List of comma separated email addresses to receive email notifications about secrets - # detected during pull request scanning. The notification is also sent to the email set - # in the committer git profile regardless of whether this variable is set or not. - # JF_EMAIL_RECEIVERS: "" \ No newline at end of file + # JF_MIN_SEVERITY: "" \ No newline at end of file From f8dd980f08b97ee0c2599855070b4cda1a97296f Mon Sep 17 00:00:00 2001 From: Michael Sverdlov Date: Thu, 31 Aug 2023 12:17:20 +0300 Subject: [PATCH 10/12] copy dir Signed-off-by: Michael Sverdlov --- go.mod | 8 +-- go.sum | 16 +++-- utils/{antUtils.go => antutils.go} | 0 utils/{antUtils_test.go => antutils_test.go} | 0 utils/io/fileutils/files.go | 68 +------------------- utils/io/fileutils/files_test.go | 3 +- utils/tests/utils.go | 7 +- utils/vcsdetails_test.go | 3 +- 8 files changed, 23 insertions(+), 82 deletions(-) rename utils/{antUtils.go => antutils.go} (100%) rename utils/{antUtils_test.go => antutils_test.go} (100%) diff --git a/go.mod b/go.mod index 8155088b2..0b3911436 100644 --- a/go.mod +++ b/go.mod @@ -3,13 +3,13 @@ module github.com/jfrog/jfrog-client-go go 1.20 require ( - github.com/ProtonMail/go-crypto v0.0.0-20230717121422-5aa5874ade95 + github.com/ProtonMail/go-crypto v0.0.0-20230828082145-3c4c8a2d2371 github.com/buger/jsonparser v1.1.1 github.com/forPelevin/gomoji v1.1.8 github.com/go-git/go-git/v5 v5.8.1 github.com/golang-jwt/jwt/v4 v4.5.0 github.com/gookit/color v1.5.4 - github.com/jfrog/build-info-go v1.9.8 + github.com/jfrog/build-info-go v1.9.9 github.com/jfrog/gofrog v1.3.0 github.com/mholt/archiver/v3 v3.5.1 github.com/stretchr/testify v1.8.4 @@ -21,7 +21,7 @@ require ( require ( dario.cat/mergo v1.0.0 // indirect - github.com/CycloneDX/cyclonedx-go v0.7.1 // indirect + github.com/CycloneDX/cyclonedx-go v0.7.2 // indirect github.com/Microsoft/go-winio v0.6.1 // indirect github.com/acomagu/bufpipe v1.0.4 // indirect github.com/andybalholm/brotli v1.0.1 // indirect @@ -57,6 +57,6 @@ require ( gopkg.in/yaml.v3 v3.0.1 // indirect ) -// replace github.com/jfrog/build-info-go => github.com/jfrog/build-info-go v1.8.9-0.20230803131422-8230595ceb86 +replace github.com/jfrog/build-info-go => github.com/jfrog/build-info-go v1.8.9-0.20230831090828-915afc4d7380 // replace github.com/jfrog/gofrog => github.com/jfrog/gofrog dev diff --git a/go.sum b/go.sum index 0709ab1aa..8cb1523fe 100644 --- a/go.sum +++ b/go.sum @@ -1,12 +1,12 @@ dario.cat/mergo v1.0.0 h1:AGCNq9Evsj31mOgNPcLyXc+4PNABt905YmuqPYYpBWk= dario.cat/mergo v1.0.0/go.mod h1:uNxQE+84aUszobStD9th8a29P2fMDhsBdgRYvZOxGmk= -github.com/CycloneDX/cyclonedx-go v0.7.1 h1:5w1SxjGm9MTMNTuRbEPyw21ObdbaagTWF/KfF0qHTRE= -github.com/CycloneDX/cyclonedx-go v0.7.1/go.mod h1:N/nrdWQI2SIjaACyyDs/u7+ddCkyl/zkNs8xFsHF2Ps= +github.com/CycloneDX/cyclonedx-go v0.7.2 h1:kKQ0t1dPOlugSIYVOMiMtFqeXI2wp/f5DBIdfux8gnQ= +github.com/CycloneDX/cyclonedx-go v0.7.2/go.mod h1:K2bA+324+Og0X84fA8HhN2X066K7Bxz4rpMQ4ZhjtSk= github.com/Microsoft/go-winio v0.5.2/go.mod h1:WpS1mjBmmwHBEWmogvA2mj8546UReBk4v8QkMxJ6pZY= github.com/Microsoft/go-winio v0.6.1 h1:9/kr64B9VUZrLm5YYwbGtUJnMgqWVOdUAXu6Migciow= github.com/Microsoft/go-winio v0.6.1/go.mod h1:LRdKpFKfdobln8UmuiYcKPot9D2v6svN5+sAH+4kjUM= -github.com/ProtonMail/go-crypto v0.0.0-20230717121422-5aa5874ade95 h1:KLq8BE0KwCL+mmXnjLWEAOYO+2l2AE4YMmqG1ZpZHBs= -github.com/ProtonMail/go-crypto v0.0.0-20230717121422-5aa5874ade95/go.mod h1:EjAoLdwvbIOoOQr3ihjnSoLZRtE8azugULFRteWMNc0= +github.com/ProtonMail/go-crypto v0.0.0-20230828082145-3c4c8a2d2371 h1:kkhsdkhsCvIsutKu5zLMgWtgh9YxGCNAw8Ad8hjwfYg= +github.com/ProtonMail/go-crypto v0.0.0-20230828082145-3c4c8a2d2371/go.mod h1:EjAoLdwvbIOoOQr3ihjnSoLZRtE8azugULFRteWMNc0= github.com/acomagu/bufpipe v1.0.4 h1:e3H4WUzM3npvo5uv95QuJM3cQspFNtFBzvJ2oNjKIDQ= github.com/acomagu/bufpipe v1.0.4/go.mod h1:mxdxdup/WdsKVreO5GpW4+M/1CE2sMG4jeGJ2sYmHc4= github.com/andybalholm/brotli v1.0.1 h1:KqhlKozYbRtJvsPrrEeXcO+N2l6NYT5A2QAFmSULpEc= @@ -51,8 +51,8 @@ github.com/gookit/color v1.5.4 h1:FZmqs7XOyGgCAxmWyPslpiok1k05wmY3SJTytgvYFs0= github.com/gookit/color v1.5.4/go.mod h1:pZJOeOS8DM43rXbp4AZo1n9zCU2qjpcRko0b6/QJi9w= github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 h1:BQSFePA1RWJOlocH6Fxy8MmwDt+yVQYULKfN0RoTN8A= github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99/go.mod h1:1lJo3i6rXxKeerYnT8Nvf0QmHCRC1n8sfWVwXF2Frvo= -github.com/jfrog/build-info-go v1.9.8 h1:D8/ga+YgQpqp/CJj2zteS4/twmSy8zvm1v9lCd2Kv1M= -github.com/jfrog/build-info-go v1.9.8/go.mod h1:t31QRpH5xUJKw8XkQlAA+Aq7aanyS1rrzpcK8xSNVts= +github.com/jfrog/build-info-go v1.8.9-0.20230831090828-915afc4d7380 h1:zfcXR/kIDe5npI+k0FdY0pWVTOLbK5kNkf0W0C7r6Gk= +github.com/jfrog/build-info-go v1.8.9-0.20230831090828-915afc4d7380/go.mod h1:QEskae5fQpjeY2PBzsjWtUQVskYSNDF2sSmw/Gx44dQ= github.com/jfrog/gofrog v1.3.0 h1:o4zgsBZE4QyDbz2M7D4K6fXPTBJht+8lE87mS9bw7Gk= github.com/jfrog/gofrog v1.3.0/go.mod h1:IFMc+V/yf7rA5WZ74CSbXe+Lgf0iApEQLxRZVzKRUR0= github.com/kevinburke/ssh_config v1.2.0 h1:x584FjTGwHzMwvHx18PXxbBVzfnxogHaAReU4gf13a4= @@ -102,11 +102,15 @@ github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXf github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/terminalstatic/go-xsd-validate v0.1.5 h1:RqpJnf6HGE2CB/lZB1A8BYguk8uRtcvYAPLCF15qguo= github.com/ulikunitz/xz v0.5.8/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= github.com/ulikunitz/xz v0.5.9 h1:RsKRIA2MO8x56wkkcd3LbtcE/uMszhb6DpRf+3uwa3I= github.com/ulikunitz/xz v0.5.9/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= github.com/xanzy/ssh-agent v0.3.3 h1:+/15pJfg/RsTxqYcX6fHqOXZwwMP+2VyYWJeWM2qQFM= github.com/xanzy/ssh-agent v0.3.3/go.mod h1:6dzNDKs0J9rVPHPhaGCukekBHKqfl+L3KghI1Bc68Uw= +github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f h1:J9EGpcZtP0E/raorCMxlFGSTBrsSlaDGf3jU/qvAE2c= +github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 h1:EzJWgHovont7NscjpAxXsDA8S8BMYve8Y5+7cuRE7R0= +github.com/xeipuuv/gojsonschema v1.2.0 h1:LhYJRs+L4fBtjZUfuSZIKGeVu0QRy8e5Xi7D17UxZ74= github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8 h1:nIPpBwaJSVYIxUFsDv3M8ofmx9yWTog9BfvIu0q41lo= github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8/go.mod h1:HUYIGzjTL3rfEspMxjDjgmT5uz5wzYJKVo23qUhYTos= github.com/xo/terminfo v0.0.0-20210125001918-ca9a967f8778 h1:QldyIu/L63oPpyvQmHgvgickp1Yw510KJOqX7H24mg8= diff --git a/utils/antUtils.go b/utils/antutils.go similarity index 100% rename from utils/antUtils.go rename to utils/antutils.go diff --git a/utils/antUtils_test.go b/utils/antutils_test.go similarity index 100% rename from utils/antUtils_test.go rename to utils/antutils_test.go diff --git a/utils/io/fileutils/files.go b/utils/io/fileutils/files.go index 149ad9b5c..b0a43fc1d 100644 --- a/utils/io/fileutils/files.go +++ b/utils/io/fileutils/files.go @@ -17,7 +17,6 @@ import ( biutils "github.com/jfrog/build-info-go/utils" gofrog "github.com/jfrog/gofrog/io" "github.com/jfrog/jfrog-client-go/utils/errorutils" - "golang.org/x/exp/slices" ) const ( @@ -417,71 +416,6 @@ type FileDetails struct { Size int64 } -func CopyFile(dst, src string) (err error) { - srcFile, err := os.Open(src) - if err != nil { - return errorutils.CheckError(err) - } - defer func() { - err = errors.Join(err, errorutils.CheckError(srcFile.Close())) - }() - fileName, _ := GetFileAndDirFromPath(src) - dstPath, err := CreateFilePath(dst, fileName) - if err != nil { - return err - } - dstFile, err := os.Create(dstPath) - if err != nil { - return errorutils.CheckError(err) - } - defer func() { - err = errors.Join(err, errorutils.CheckError(dstFile.Close())) - }() - _, err = io.Copy(dstFile, srcFile) - return errorutils.CheckError(err) -} - -// Copy directory content from one path to another. -// includeDirs means to copy also the dirs if presented in the src folder. -// excludeNames - Skip files/dirs in the src folder that match names in provided slice. ONLY excludes first layer (only in src folder). -func CopyDir(fromPath, toPath string, includeDirs bool, excludeNames []string) error { - err := CreateDirIfNotExist(toPath) - if err != nil { - return err - } - - files, err := ListFiles(fromPath, includeDirs) - if err != nil { - return err - } - - for _, v := range files { - // Skip if excluded - if slices.Contains(excludeNames, filepath.Base(v)) { - continue - } - - dir, err := IsDirExists(v, false) - if err != nil { - return err - } - - if dir { - toPath := toPath + GetFileSeparator() + filepath.Base(v) - err := CopyDir(v, toPath, true, nil) - if err != nil { - return err - } - continue - } - err = CopyFile(toPath, v) - if err != nil { - return err - } - } - return err -} - // Removing the provided path from the filesystem func RemovePath(testPath string) error { if _, err := os.Stat(testPath); err == nil { @@ -496,7 +430,7 @@ func RemovePath(testPath string) error { // Renaming from old path to new path. func RenamePath(oldPath, newPath string) error { - err := CopyDir(oldPath, newPath, true, nil) + err := biutils.CopyDir(oldPath, newPath, true, nil) if err != nil { return errors.New("Error copying directory: " + oldPath + "to" + newPath + err.Error()) } diff --git a/utils/io/fileutils/files_test.go b/utils/io/fileutils/files_test.go index fff171a5b..cf27e7a51 100644 --- a/utils/io/fileutils/files_test.go +++ b/utils/io/fileutils/files_test.go @@ -1,6 +1,7 @@ package fileutils import ( + biutils "github.com/jfrog/build-info-go/utils" "github.com/jfrog/jfrog-client-go/utils/io" "os" "path/filepath" @@ -262,7 +263,7 @@ func TestRemoveDirContents(t *testing.T) { defer func() { assert.NoError(t, RemoveTempDir(tmpDirPath)) }() - err = CopyDir(filepath.Join("testdata", "removedircontents"), tmpDirPath, true, nil) + err = biutils.CopyDir(filepath.Join("testdata", "removedircontents"), tmpDirPath, true, nil) assert.NoError(t, err) // Run the function diff --git a/utils/tests/utils.go b/utils/tests/utils.go index 3bfef9a3a..515c0a006 100644 --- a/utils/tests/utils.go +++ b/utils/tests/utils.go @@ -3,6 +3,7 @@ package tests import ( "bufio" "errors" + biutils "github.com/jfrog/build-info-go/utils" "github.com/jfrog/jfrog-client-go/utils/io/fileutils" "github.com/jfrog/jfrog-client-go/utils/log" "github.com/stretchr/testify/assert" @@ -107,13 +108,13 @@ func exitOnErr(err error) { func InitVcsSubmoduleTestDir(t *testing.T, srcPath, tmpDir string) (submodulePath string) { var err error - assert.NoError(t, fileutils.CopyDir(srcPath, tmpDir, true, nil)) + assert.NoError(t, biutils.CopyDir(srcPath, tmpDir, true, nil)) if found, err := fileutils.IsDirExists(filepath.Join(tmpDir, "gitdata"), false); found { assert.NoError(t, err) assert.NoError(t, fileutils.RenamePath(filepath.Join(tmpDir, "gitdata"), filepath.Join(tmpDir, ".git"))) } submoduleDst := filepath.Join(tmpDir, "subdir", "submodule") - assert.NoError(t, fileutils.CopyFile(submoduleDst, filepath.Join(tmpDir, "gitSubmoduleData"))) + assert.NoError(t, biutils.CopyFile(submoduleDst, filepath.Join(tmpDir, "gitSubmoduleData"))) assert.NoError(t, fileutils.MoveFile(filepath.Join(submoduleDst, "gitSubmoduleData"), filepath.Join(submoduleDst, ".git"))) submodulePath, err = filepath.Abs(submoduleDst) assert.NoError(t, err) @@ -122,7 +123,7 @@ func InitVcsSubmoduleTestDir(t *testing.T, srcPath, tmpDir string) (submodulePat func InitVcsWorktreeTestDir(t *testing.T, srcPath, tmpDir string) (worktreePath string) { var err error - assert.NoError(t, fileutils.CopyDir(srcPath, tmpDir, true, nil)) + assert.NoError(t, biutils.CopyDir(srcPath, tmpDir, true, nil)) if found, err := fileutils.IsDirExists(filepath.Join(tmpDir, "gitdata"), false); found { assert.NoError(t, err) assert.NoError(t, fileutils.RenamePath(filepath.Join(tmpDir, "gitdata"), filepath.Join(tmpDir, "bare.git"))) diff --git a/utils/vcsdetails_test.go b/utils/vcsdetails_test.go index 9720ef35c..d60499681 100644 --- a/utils/vcsdetails_test.go +++ b/utils/vcsdetails_test.go @@ -1,6 +1,7 @@ package utils import ( + biutils "github.com/jfrog/build-info-go/utils" testsutils "github.com/jfrog/jfrog-client-go/utils/tests" "github.com/stretchr/testify/assert" "path/filepath" @@ -42,7 +43,7 @@ func TestVcsDetails(t *testing.T) { func initVcsTestDir(t *testing.T, srcPath, tmpDir string) (projectPath string) { var err error - assert.NoError(t, fileutils.CopyDir(srcPath, tmpDir, true, nil)) + assert.NoError(t, biutils.CopyDir(srcPath, tmpDir, true, nil)) if found, err := fileutils.IsDirExists(filepath.Join(tmpDir, "gitdata"), false); found { assert.NoError(t, err) assert.NoError(t, fileutils.RenamePath(filepath.Join(tmpDir, "gitdata"), filepath.Join(tmpDir, ".git"))) From 814128405937296dde9ea283eeb1dfb147dea67b Mon Sep 17 00:00:00 2001 From: Michael Sverdlov Date: Thu, 31 Aug 2023 18:17:20 +0300 Subject: [PATCH 11/12] copy dir Signed-off-by: Michael Sverdlov --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 0b3911436..a9c17268c 100644 --- a/go.mod +++ b/go.mod @@ -57,6 +57,6 @@ require ( gopkg.in/yaml.v3 v3.0.1 // indirect ) -replace github.com/jfrog/build-info-go => github.com/jfrog/build-info-go v1.8.9-0.20230831090828-915afc4d7380 +replace github.com/jfrog/build-info-go => github.com/jfrog/build-info-go v1.8.9-0.20230831151231-e5e7bd035ddc // replace github.com/jfrog/gofrog => github.com/jfrog/gofrog dev diff --git a/go.sum b/go.sum index 8cb1523fe..0af3b5b73 100644 --- a/go.sum +++ b/go.sum @@ -51,8 +51,8 @@ github.com/gookit/color v1.5.4 h1:FZmqs7XOyGgCAxmWyPslpiok1k05wmY3SJTytgvYFs0= github.com/gookit/color v1.5.4/go.mod h1:pZJOeOS8DM43rXbp4AZo1n9zCU2qjpcRko0b6/QJi9w= github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 h1:BQSFePA1RWJOlocH6Fxy8MmwDt+yVQYULKfN0RoTN8A= github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99/go.mod h1:1lJo3i6rXxKeerYnT8Nvf0QmHCRC1n8sfWVwXF2Frvo= -github.com/jfrog/build-info-go v1.8.9-0.20230831090828-915afc4d7380 h1:zfcXR/kIDe5npI+k0FdY0pWVTOLbK5kNkf0W0C7r6Gk= -github.com/jfrog/build-info-go v1.8.9-0.20230831090828-915afc4d7380/go.mod h1:QEskae5fQpjeY2PBzsjWtUQVskYSNDF2sSmw/Gx44dQ= +github.com/jfrog/build-info-go v1.8.9-0.20230831151231-e5e7bd035ddc h1:pqu82clhPKyUKJcljMuxYa+kviaWnHycLNCLqZZNl30= +github.com/jfrog/build-info-go v1.8.9-0.20230831151231-e5e7bd035ddc/go.mod h1:QEskae5fQpjeY2PBzsjWtUQVskYSNDF2sSmw/Gx44dQ= github.com/jfrog/gofrog v1.3.0 h1:o4zgsBZE4QyDbz2M7D4K6fXPTBJht+8lE87mS9bw7Gk= github.com/jfrog/gofrog v1.3.0/go.mod h1:IFMc+V/yf7rA5WZ74CSbXe+Lgf0iApEQLxRZVzKRUR0= github.com/kevinburke/ssh_config v1.2.0 h1:x584FjTGwHzMwvHx18PXxbBVzfnxogHaAReU4gf13a4= From 5cc828c827ae470d022a7f231d7f61a6ea8bc6ac Mon Sep 17 00:00:00 2001 From: Michael Sverdlov Date: Thu, 31 Aug 2023 18:27:43 +0300 Subject: [PATCH 12/12] copy dir Signed-off-by: Michael Sverdlov --- .github/workflows/tests.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 3f3209ee9..e5afb1a4f 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -51,8 +51,8 @@ jobs: fail-fast: false matrix: suite: [ artifactory, access ] - os: [ ubuntu-latest, macos-latest, windows-latest ] - runs-on: ${{ matrix.os }} + os: [ ubuntu, windows, macos ] + runs-on: ${{ matrix.os }}-latest steps: - name: Install Go uses: actions/setup-go@v3 @@ -90,8 +90,8 @@ jobs: fail-fast: false matrix: suite: [ distribution, xray ] - os: [ ubuntu-latest, macos-latest, windows-latest ] - runs-on: ${{ matrix.os }} + os: [ ubuntu, windows, macos ] + runs-on: ${{ matrix.os }}-latest steps: - name: Checkout code uses: actions/checkout@v3