Check for go.mod files in subdirectories (#39)

.github/workflows/go.yml

@@ -26,7 +26,7 @@ jobs:
     - name: Set up Go 1.x
       uses: actions/setup-go@v2
       with:
-        go-version: 1.19
+        go-version: 1.21
     - run: bundle config --local path $PWD/vendor/bundle
 
     - name: Set up Node

cmd/deplist/deplist.go

@@ -20,7 +20,7 @@ func main() {
 	}
 
 	if flag.Args() == nil || len(flag.Args()) == 0 {
-		fmt.Println("Not path to scan was specified, i.e. deplist /tmp/files/")
+		fmt.Println("No path to scan was specified, i.e. deplist /tmp/files/")
 		return
 	}
 
@@ -33,9 +33,7 @@ func main() {
 
 	if *deptypePtr == -1 {
 		for _, dep := range deps {
-			version := dep.Version
-
-			inst, _ := purl.FromString(fmt.Sprintf("pkg:%s/%s@%s", deplist.GetLanguageStr(dep.DepType), dep.Path, version))
+			inst, _ := purl.FromString(dep.ToString())
 			fmt.Println(inst)
 		}
 	} else {

dependencies.go

@@ -1,5 +1,9 @@
 package deplist
 
+import (
+	"fmt"
+)
+
 // Bitmask type allows easy tagging of what langs there are
 type Bitmask uint32
 
@@ -18,3 +22,23 @@ func (f *Bitmask) DepFoundAddFlag(flag Bitmask) { *f |= flag }
 
 // DepFoundHasFlag deteremine if bitmask has a lang type
 func (f Bitmask) DepFoundHasFlag(flag Bitmask) bool { return f&flag != 0 }
+
+func (d *Dependency) ToString() string {
+	return fmt.Sprintf("pkg:%s/%s@%s", GetLanguageStr(d.DepType), d.Path, d.Version)
+}
+
+// GetLanguageStr returns from a bitmask return the ecosystem name
+func GetLanguageStr(bm Bitmask) string {
+	if bm&LangGolang != 0 {
+		return "go"
+	} else if bm&LangJava != 0 {
+		return "mvn"
+	} else if bm&LangNodeJS != 0 {
+		return "npm"
+	} else if bm&LangPython != 0 {
+		return "pypi"
+	} else if bm&LangRuby != 0 {
+		return "gem"
+	}
+	return "unknown"
+}

deplist.go

@@ -2,7 +2,6 @@ package deplist
 
 import (
 	"fmt"
-	"io/ioutil"
 	"os"
 	"os/exec"
 	"path/filepath"
@@ -40,22 +39,6 @@ func init() {
 	}
 }
 
-// GetLanguageStr returns from a bitmask return the ecosystem name
-func GetLanguageStr(bm Bitmask) string {
-	if bm&LangGolang != 0 {
-		return "go"
-	} else if bm&LangJava != 0 {
-		return "mvn"
-	} else if bm&LangNodeJS != 0 {
-		return "npm"
-	} else if bm&LangPython != 0 {
-		return "pypi"
-	} else if bm&LangRuby != 0 {
-		return "gem"
-	}
-	return "unknown"
-}
-
 type Discovered struct {
 	deps       []Dependency
 	foundTypes Bitmask
@@ -89,7 +72,7 @@ func getDeps(fullPath string) ([]Dependency, Bitmask, error) {
 	}
 
 	pomPath := filepath.Join(fullPath, "pom.xml")
-	goPath := filepath.Join(fullPath, "go.mod")
+	// goPath := filepath.Join(fullPath, "go.mod")
 	goPkgPath := filepath.Join(fullPath, "Gopkg.lock")
 	glidePath := filepath.Join(fullPath, "glide.lock")
 	rubyPath := filepath.Join(fullPath, "Gemfile") // Later we translate Gemfile.lock -> Gemfile to handle both cases
@@ -111,8 +94,29 @@ func getDeps(fullPath string) ([]Dependency, Bitmask, error) {
 			// Two checks, one for filenames and the second switch for full
 			// paths. Useful if we're looking for top of repo
 
-			switch filename := info.Name(); filename {
-			// for now only go for yarn and npm
+			// comparisons here are made against the filename only, not full path
+			// so matches will be found at any level of the file tree, not just top-level
+			filename := info.Name()
+			switch filename {
+			case "go.mod":
+				pkgs, err := scan.GetGolangDeps(path)
+				if err != nil {
+					return err
+				}
+
+				if len(pkgs) > 0 {
+					discovered.foundTypes.DepFoundAddFlag(LangGolang)
+				}
+
+				for path, goPkg := range pkgs {
+					d := Dependency{
+						DepType: LangGolang,
+						Path:    path,
+						Files:   goPkg.Gofiles,
+						Version: goPkg.Version,
+					}
+					discovered.deps = append(discovered.deps, d)
+				}
 			case "package-lock.json":
 				// if theres not a yarn.lock fall thru
 				if _, err := os.Stat(
@@ -198,26 +202,9 @@ func getDeps(fullPath string) ([]Dependency, Bitmask, error) {
 			// but also avoid double-handling, i.e. scanning once for each file
 			path = strings.Replace(path, "Gemfile.lock", "Gemfile", 1)
 
+			// comparisons here are against the full filepath, so will not match if
+			// these filesames are found in subdirectories, only the top level
 			switch path {
-			case goPath: // just support the top level go.mod for now
-				pkgs, err := scan.GetGolangDeps(path)
-				if err != nil {
-					return err
-				}
-
-				if len(pkgs) > 0 {
-					discovered.foundTypes.DepFoundAddFlag(LangGolang)
-				}
-
-				for path, goPkg := range pkgs {
-					d := Dependency{
-						DepType: LangGolang,
-						Path:    path,
-						Files:   goPkg.Gofiles,
-						Version: goPkg.Version,
-					}
-					discovered.deps = append(discovered.deps, d)
-				}
 			case goPkgPath:
 				pkgs, err := scan.GetGoPkgDeps(path)
 				if err != nil {
@@ -285,7 +272,6 @@ func getDeps(fullPath string) ([]Dependency, Bitmask, error) {
 		}
 		return nil
 	})
-
 	if err != nil {
 		return nil, 0, err // should't matter
 	}
@@ -296,7 +282,7 @@ func getDeps(fullPath string) ([]Dependency, Bitmask, error) {
 // findBaseDir walks a directory tree through empty subdirs til it finds a directory with content
 func findBaseDir(fullPath string) (string, error) {
 	log.Debugf("Checking %s", fullPath)
-	files, err := ioutil.ReadDir(fullPath)
+	files, err := os.ReadDir(fullPath)
 	if err != nil {
 		return "", fmt.Errorf("Could not read: %s", err)
 	}
@@ -321,9 +307,27 @@ func GetDeps(fullPath string) ([]Dependency, Bitmask, error) {
 	// but ignore any new errors
 	if len(deps) == 0 {
 		fullPath = filepath.Join(fullPath, "src")
-		log.Debugf("Checking %s", fullPath)
-		deps, foundTypes, _ = getDeps(fullPath)
+		if _, err := os.Stat(fullPath); err != nil {
+			log.Debugf("No deps found, trying %s", fullPath)
+			deps, foundTypes, _ = getDeps(fullPath)
+		}
 	}
 
-	return deps, foundTypes, err
+	// de-duplicate
+	unique := removeDuplicates(deps)
+
+	return unique, foundTypes, err
+}
+
+func removeDuplicates(deps []Dependency) []Dependency {
+	seen := map[string]bool{}
+	filtered := []Dependency{}
+	for _, dep := range deps {
+		key := dep.ToString()
+		if _, ok := seen[key]; !ok {
+			seen[key] = true
+			filtered = append(filtered, dep)
+		}
+	}
+	return filtered
 }

deplist_test.go

@@ -2,7 +2,6 @@ package deplist
 
 import (
 	"fmt"
-	"io/ioutil"
 	"os"
 	"path/filepath"
 	"testing"
@@ -25,9 +24,11 @@ func BuildWant() []Dependency {
 		"golang.org/x/text/unicode",
 		"internal/abi",
 		"internal/bytealg",
+		"internal/coverage/rtcov",
 		"internal/cpu",
 		"internal/fmtsort",
 		"internal/goarch",
+		"internal/godebugs",
 		"internal/goexperiment",
 		"internal/goos",
 		"internal/itoa",
@@ -273,20 +274,20 @@ func BuildWant() []Dependency {
 	}
 
 	pythonSet := []Dependency{
-		Dependency{DepType: LangPython, Path: "cotyledon"},
-		Dependency{DepType: LangPython, Path: "Flask"},
-		Dependency{DepType: LangPython, Path: "kuryr-lib"},
-		Dependency{DepType: LangPython, Path: "docutils"},
-		Dependency{DepType: LangPython, Path: "python-dateutil"},
-		Dependency{DepType: LangPython, Path: "unittest2", Version: "0.5.1"},
-		Dependency{DepType: LangPython, Path: "cryptography", Version: "2.3.0"},
-		Dependency{DepType: LangPython, Path: "suds-py3"},
-		Dependency{DepType: LangPython, Path: "suds"},
-		Dependency{DepType: LangPython, Path: "git+https://github.com/candlepin/subscription-manager#egg=subscription_manager"},
-		Dependency{DepType: LangPython, Path: "git+https://github.com/candlepin/python-iniparse#egg=iniparse"},
-		Dependency{DepType: LangPython, Path: "iniparse"},
-		Dependency{DepType: LangPython, Path: "requests"},
-		Dependency{DepType: LangPython, Path: "m2crypto"},
+		{DepType: LangPython, Path: "cotyledon"},
+		{DepType: LangPython, Path: "Flask"},
+		{DepType: LangPython, Path: "kuryr-lib"},
+		{DepType: LangPython, Path: "docutils"},
+		{DepType: LangPython, Path: "python-dateutil"},
+		{DepType: LangPython, Path: "unittest2", Version: "0.5.1"},
+		{DepType: LangPython, Path: "cryptography", Version: "2.3.0"},
+		{DepType: LangPython, Path: "suds-py3"},
+		{DepType: LangPython, Path: "suds"},
+		{DepType: LangPython, Path: "git+https://github.com/candlepin/subscription-manager#egg=subscription_manager"},
+		{DepType: LangPython, Path: "git+https://github.com/candlepin/python-iniparse#egg=iniparse"},
+		{DepType: LangPython, Path: "iniparse"},
+		{DepType: LangPython, Path: "requests"},
+		{DepType: LangPython, Path: "m2crypto"},
 	}
 
 	for _, n := range golangPaths {
@@ -418,14 +419,14 @@ func TestFindBaseDir(t *testing.T) {
 	}
 
 	dirpath := filepath.Join(top, "baz")
-	os.MkdirAll(dirpath, 0755)
+	os.MkdirAll(dirpath, 0o755)
 	tests[1] = TestCase{
 		Input:    dirpath,
 		Expected: dirpath,
 		Err:      false,
 	}
 
-	tempFile, err := ioutil.TempFile(top, "bar")
+	tempFile, err := os.CreateTemp(top, "bar")
 	if err != nil {
 		t.Error(err)
 	}
@@ -436,7 +437,7 @@ func TestFindBaseDir(t *testing.T) {
 	}
 
 	dirpath = filepath.Join(top, "foo/bar/foo/bar/foo/bar")
-	err = os.MkdirAll(dirpath, 0755)
+	err = os.MkdirAll(dirpath, 0o755)
 	if err != nil {
 		t.Error(err)
 	}
@@ -448,11 +449,11 @@ func TestFindBaseDir(t *testing.T) {
 
 	top = t.TempDir()
 	dirpath = filepath.Join(top, "foo/bar/foo/bar/foo/bar")
-	err = os.MkdirAll(dirpath, 0755)
+	err = os.MkdirAll(dirpath, 0o755)
 	if err != nil {
 		t.Error(err)
 	}
-	tempFile, err = ioutil.TempFile(filepath.Join(top, "foo/bar/foo"), "baz")
+	_, err = os.CreateTemp(filepath.Join(top, "foo/bar/foo"), "baz")
 	if err != nil {
 		t.Error(err)
 	}

go.mod

@@ -1,6 +1,6 @@
 module github.com/RedHatProductSecurity/deplist
 
-go 1.17
+go 1.21
 
 require (
 	github.com/BurntSushi/toml v0.4.1