From deac06f25ff4ae1a6baa7ca63484c62aef4c2cd6 Mon Sep 17 00:00:00 2001 From: sfowl Date: Mon, 9 Dec 2024 14:49:18 +1000 Subject: [PATCH] Check for go.mod files in subdirectories (#39) --- .github/workflows/go.yml | 2 +- cmd/deplist/deplist.go | 6 +-- dependencies.go | 24 +++++++++++ deplist.go | 92 +++++++++++++++++++++------------------- deplist_test.go | 41 +++++++++--------- go.mod | 2 +- 6 files changed, 97 insertions(+), 70 deletions(-) diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml index 4de1b07..19778fd 100644 --- a/.github/workflows/go.yml +++ b/.github/workflows/go.yml @@ -26,7 +26,7 @@ jobs: - name: Set up Go 1.x uses: actions/setup-go@v2 with: - go-version: 1.19 + go-version: 1.21 - run: bundle config --local path $PWD/vendor/bundle - name: Set up Node diff --git a/cmd/deplist/deplist.go b/cmd/deplist/deplist.go index bb79c79..c57d3f3 100644 --- a/cmd/deplist/deplist.go +++ b/cmd/deplist/deplist.go @@ -20,7 +20,7 @@ func main() { } if flag.Args() == nil || len(flag.Args()) == 0 { - fmt.Println("Not path to scan was specified, i.e. deplist /tmp/files/") + fmt.Println("No path to scan was specified, i.e. deplist /tmp/files/") return } @@ -33,9 +33,7 @@ func main() { if *deptypePtr == -1 { for _, dep := range deps { - version := dep.Version - - inst, _ := purl.FromString(fmt.Sprintf("pkg:%s/%s@%s", deplist.GetLanguageStr(dep.DepType), dep.Path, version)) + inst, _ := purl.FromString(dep.ToString()) fmt.Println(inst) } } else { diff --git a/dependencies.go b/dependencies.go index 3cf8c2e..770aca2 100644 --- a/dependencies.go +++ b/dependencies.go @@ -1,5 +1,9 @@ package deplist +import ( + "fmt" +) + // Bitmask type allows easy tagging of what langs there are type Bitmask uint32 @@ -18,3 +22,23 @@ func (f *Bitmask) DepFoundAddFlag(flag Bitmask) { *f |= flag } // DepFoundHasFlag deteremine if bitmask has a lang type func (f Bitmask) DepFoundHasFlag(flag Bitmask) bool { return f&flag != 0 } + +func (d *Dependency) ToString() string { + return fmt.Sprintf("pkg:%s/%s@%s", GetLanguageStr(d.DepType), d.Path, d.Version) +} + +// GetLanguageStr returns from a bitmask return the ecosystem name +func GetLanguageStr(bm Bitmask) string { + if bm&LangGolang != 0 { + return "go" + } else if bm&LangJava != 0 { + return "mvn" + } else if bm&LangNodeJS != 0 { + return "npm" + } else if bm&LangPython != 0 { + return "pypi" + } else if bm&LangRuby != 0 { + return "gem" + } + return "unknown" +} diff --git a/deplist.go b/deplist.go index 08e3f27..5d09061 100644 --- a/deplist.go +++ b/deplist.go @@ -2,7 +2,6 @@ package deplist import ( "fmt" - "io/ioutil" "os" "os/exec" "path/filepath" @@ -40,22 +39,6 @@ func init() { } } -// GetLanguageStr returns from a bitmask return the ecosystem name -func GetLanguageStr(bm Bitmask) string { - if bm&LangGolang != 0 { - return "go" - } else if bm&LangJava != 0 { - return "mvn" - } else if bm&LangNodeJS != 0 { - return "npm" - } else if bm&LangPython != 0 { - return "pypi" - } else if bm&LangRuby != 0 { - return "gem" - } - return "unknown" -} - type Discovered struct { deps []Dependency foundTypes Bitmask @@ -89,7 +72,7 @@ func getDeps(fullPath string) ([]Dependency, Bitmask, error) { } pomPath := filepath.Join(fullPath, "pom.xml") - goPath := filepath.Join(fullPath, "go.mod") + // goPath := filepath.Join(fullPath, "go.mod") goPkgPath := filepath.Join(fullPath, "Gopkg.lock") glidePath := filepath.Join(fullPath, "glide.lock") rubyPath := filepath.Join(fullPath, "Gemfile") // Later we translate Gemfile.lock -> Gemfile to handle both cases @@ -111,8 +94,29 @@ func getDeps(fullPath string) ([]Dependency, Bitmask, error) { // Two checks, one for filenames and the second switch for full // paths. Useful if we're looking for top of repo - switch filename := info.Name(); filename { - // for now only go for yarn and npm + // comparisons here are made against the filename only, not full path + // so matches will be found at any level of the file tree, not just top-level + filename := info.Name() + switch filename { + case "go.mod": + pkgs, err := scan.GetGolangDeps(path) + if err != nil { + return err + } + + if len(pkgs) > 0 { + discovered.foundTypes.DepFoundAddFlag(LangGolang) + } + + for path, goPkg := range pkgs { + d := Dependency{ + DepType: LangGolang, + Path: path, + Files: goPkg.Gofiles, + Version: goPkg.Version, + } + discovered.deps = append(discovered.deps, d) + } case "package-lock.json": // if theres not a yarn.lock fall thru if _, err := os.Stat( @@ -198,26 +202,9 @@ func getDeps(fullPath string) ([]Dependency, Bitmask, error) { // but also avoid double-handling, i.e. scanning once for each file path = strings.Replace(path, "Gemfile.lock", "Gemfile", 1) + // comparisons here are against the full filepath, so will not match if + // these filesames are found in subdirectories, only the top level switch path { - case goPath: // just support the top level go.mod for now - pkgs, err := scan.GetGolangDeps(path) - if err != nil { - return err - } - - if len(pkgs) > 0 { - discovered.foundTypes.DepFoundAddFlag(LangGolang) - } - - for path, goPkg := range pkgs { - d := Dependency{ - DepType: LangGolang, - Path: path, - Files: goPkg.Gofiles, - Version: goPkg.Version, - } - discovered.deps = append(discovered.deps, d) - } case goPkgPath: pkgs, err := scan.GetGoPkgDeps(path) if err != nil { @@ -285,7 +272,6 @@ func getDeps(fullPath string) ([]Dependency, Bitmask, error) { } return nil }) - if err != nil { return nil, 0, err // should't matter } @@ -296,7 +282,7 @@ func getDeps(fullPath string) ([]Dependency, Bitmask, error) { // findBaseDir walks a directory tree through empty subdirs til it finds a directory with content func findBaseDir(fullPath string) (string, error) { log.Debugf("Checking %s", fullPath) - files, err := ioutil.ReadDir(fullPath) + files, err := os.ReadDir(fullPath) if err != nil { return "", fmt.Errorf("Could not read: %s", err) } @@ -321,9 +307,27 @@ func GetDeps(fullPath string) ([]Dependency, Bitmask, error) { // but ignore any new errors if len(deps) == 0 { fullPath = filepath.Join(fullPath, "src") - log.Debugf("Checking %s", fullPath) - deps, foundTypes, _ = getDeps(fullPath) + if _, err := os.Stat(fullPath); err != nil { + log.Debugf("No deps found, trying %s", fullPath) + deps, foundTypes, _ = getDeps(fullPath) + } } - return deps, foundTypes, err + // de-duplicate + unique := removeDuplicates(deps) + + return unique, foundTypes, err +} + +func removeDuplicates(deps []Dependency) []Dependency { + seen := map[string]bool{} + filtered := []Dependency{} + for _, dep := range deps { + key := dep.ToString() + if _, ok := seen[key]; !ok { + seen[key] = true + filtered = append(filtered, dep) + } + } + return filtered } diff --git a/deplist_test.go b/deplist_test.go index afbc962..1d38523 100644 --- a/deplist_test.go +++ b/deplist_test.go @@ -2,7 +2,6 @@ package deplist import ( "fmt" - "io/ioutil" "os" "path/filepath" "testing" @@ -25,9 +24,11 @@ func BuildWant() []Dependency { "golang.org/x/text/unicode", "internal/abi", "internal/bytealg", + "internal/coverage/rtcov", "internal/cpu", "internal/fmtsort", "internal/goarch", + "internal/godebugs", "internal/goexperiment", "internal/goos", "internal/itoa", @@ -273,20 +274,20 @@ func BuildWant() []Dependency { } pythonSet := []Dependency{ - Dependency{DepType: LangPython, Path: "cotyledon"}, - Dependency{DepType: LangPython, Path: "Flask"}, - Dependency{DepType: LangPython, Path: "kuryr-lib"}, - Dependency{DepType: LangPython, Path: "docutils"}, - Dependency{DepType: LangPython, Path: "python-dateutil"}, - Dependency{DepType: LangPython, Path: "unittest2", Version: "0.5.1"}, - Dependency{DepType: LangPython, Path: "cryptography", Version: "2.3.0"}, - Dependency{DepType: LangPython, Path: "suds-py3"}, - Dependency{DepType: LangPython, Path: "suds"}, - Dependency{DepType: LangPython, Path: "git+https://github.com/candlepin/subscription-manager#egg=subscription_manager"}, - Dependency{DepType: LangPython, Path: "git+https://github.com/candlepin/python-iniparse#egg=iniparse"}, - Dependency{DepType: LangPython, Path: "iniparse"}, - Dependency{DepType: LangPython, Path: "requests"}, - Dependency{DepType: LangPython, Path: "m2crypto"}, + {DepType: LangPython, Path: "cotyledon"}, + {DepType: LangPython, Path: "Flask"}, + {DepType: LangPython, Path: "kuryr-lib"}, + {DepType: LangPython, Path: "docutils"}, + {DepType: LangPython, Path: "python-dateutil"}, + {DepType: LangPython, Path: "unittest2", Version: "0.5.1"}, + {DepType: LangPython, Path: "cryptography", Version: "2.3.0"}, + {DepType: LangPython, Path: "suds-py3"}, + {DepType: LangPython, Path: "suds"}, + {DepType: LangPython, Path: "git+https://github.com/candlepin/subscription-manager#egg=subscription_manager"}, + {DepType: LangPython, Path: "git+https://github.com/candlepin/python-iniparse#egg=iniparse"}, + {DepType: LangPython, Path: "iniparse"}, + {DepType: LangPython, Path: "requests"}, + {DepType: LangPython, Path: "m2crypto"}, } for _, n := range golangPaths { @@ -418,14 +419,14 @@ func TestFindBaseDir(t *testing.T) { } dirpath := filepath.Join(top, "baz") - os.MkdirAll(dirpath, 0755) + os.MkdirAll(dirpath, 0o755) tests[1] = TestCase{ Input: dirpath, Expected: dirpath, Err: false, } - tempFile, err := ioutil.TempFile(top, "bar") + tempFile, err := os.CreateTemp(top, "bar") if err != nil { t.Error(err) } @@ -436,7 +437,7 @@ func TestFindBaseDir(t *testing.T) { } dirpath = filepath.Join(top, "foo/bar/foo/bar/foo/bar") - err = os.MkdirAll(dirpath, 0755) + err = os.MkdirAll(dirpath, 0o755) if err != nil { t.Error(err) } @@ -448,11 +449,11 @@ func TestFindBaseDir(t *testing.T) { top = t.TempDir() dirpath = filepath.Join(top, "foo/bar/foo/bar/foo/bar") - err = os.MkdirAll(dirpath, 0755) + err = os.MkdirAll(dirpath, 0o755) if err != nil { t.Error(err) } - tempFile, err = ioutil.TempFile(filepath.Join(top, "foo/bar/foo"), "baz") + _, err = os.CreateTemp(filepath.Join(top, "foo/bar/foo"), "baz") if err != nil { t.Error(err) } diff --git a/go.mod b/go.mod index 0fcb882..893be1a 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/RedHatProductSecurity/deplist -go 1.17 +go 1.21 require ( github.com/BurntSushi/toml v0.4.1