python: add initial (#14)

* python: add initial code to scan requirements.txt file

Signed-off-by: mcoops <[email protected]>

deplist.go

@@ -52,6 +52,7 @@ func GetDeps(fullPath string) ([]Dependency, Bitmask, error) {
 	pomPath := filepath.Join(fullPath, "pom.xml")
 	goPath := filepath.Join(fullPath, "go.mod")
 	rubyPath := filepath.Join(fullPath, "Gemfile.lock")
+	pythonPath := filepath.Join(fullPath, "requirements.txt")
 
 	// point at the parent repo, but can't assume where the indicators will be
 	err := filepath.Walk(fullPath, func(path string, info os.FileInfo, err error) error {
@@ -87,6 +88,7 @@ func GetDeps(fullPath string) ([]Dependency, Bitmask, error) {
 							DepType: LangNodeJS,
 							Path:    name,
 							Version: strings.Replace(version, "v", "", 1),
+							Files:   []string{},
 						})
 				}
 			}
@@ -128,6 +130,7 @@ func GetDeps(fullPath string) ([]Dependency, Bitmask, error) {
 							DepType: LangJava,
 							Path:    name,
 							Version: strings.Replace(version, "v", "", 1),
+							Files:   []string{},
 						})
 				}
 			case rubyPath:
@@ -146,6 +149,26 @@ func GetDeps(fullPath string) ([]Dependency, Bitmask, error) {
 							DepType: LangRuby,
 							Path:    name,
 							Version: strings.Replace(version, "v", "", 1),
+							Files:   []string{},
+						})
+				}
+			case pythonPath:
+				pkgs, err := scan.GetPythonDeps(path)
+				if err != nil {
+					return err
+				}
+
+				if len(pkgs) > 0 {
+					foundTypes.DepFoundAddFlag(LangPython)
+				}
+
+				for name, version := range pkgs {
+					deps = append(deps,
+						Dependency{
+							DepType: LangPython,
+							Path:    name,
+							Version: version,
+							Files:   []string{},
 						})
 				}
 			}

deplist_test.go

@@ -183,6 +183,13 @@ func BuildWant() []Dependency {
 		"addressable",
 	}
 
+	pythonSet := []string{
+		"cotyledon",
+		"Flask",
+		"kuryr-lib",
+		"cryptography",
+	}
+
 	for _, n := range golangPaths {
 		d := Dependency{
 			DepType: 1,
@@ -219,6 +226,18 @@ func BuildWant() []Dependency {
 		}
 		deps = append(deps, d)
 	}
+
+	for _, n := range pythonSet {
+		d := Dependency{
+			DepType: LangPython,
+			Path:    n,
+		}
+		deps = append(deps, d)
+	}
+
+	end = len(deps) - 1 // get the cryptography ver
+	deps[end].Version = "2.3.0"
+
 	return deps
 }
 
@@ -227,8 +246,8 @@ func TestGetDeps(t *testing.T) {
 
 	got, gotBitmask, _ := GetDeps("test/testRepo")
 
-	if gotBitmask != 23 {
-		t.Errorf("GotBitmask() != 7; got: %d", gotBitmask)
+	if gotBitmask != 31 {
+		t.Errorf("GotBitmask() != 31; got: %d", gotBitmask)
 	}
 
 	// iterate thru and compare

internal/scan/python.go

@@ -0,0 +1,59 @@
+package scan
+
+import (
+	"bufio"
+	"os"
+	"regexp"
+	"strings"
+)
+
+// Account for >, <, >=, <=, ==, !=, ~= and *
+var /* const */ re = regexp.MustCompile(`[<>!~*]+`)
+
+func max(x, y int) int {
+	if x > y {
+		return x
+	}
+	return y
+}
+
+func GetPythonDeps(path string) (map[string]string, error) {
+	gathered := make(map[string]string)
+
+	file, err := os.Open(path)
+
+	if err != nil {
+		return nil, err
+	}
+
+	defer file.Close()
+
+	scanner := bufio.NewScanner(file)
+
+	for scanner.Scan() {
+		line := scanner.Text()
+
+		// skip comments
+		if strings.HasPrefix(line, "#") || line == "" {
+			continue
+		}
+
+		// easy case, elasticsearch-curator==5.8.1
+		// record name and version, only for ==
+		idx := strings.LastIndex(line, "==")
+		if idx > 0 {
+			gathered[line[:idx]] = line[idx+2:]
+			continue
+		}
+
+		// every other permitation just use the name as we can't guarantee
+		// the version, just grab the name using first occurance
+		match := re.FindStringIndex(line)
+
+		if match != nil {
+			gathered[line[:match[0]]] = ""
+		}
+	}
+
+	return gathered, nil
+}

test/testRepo/requirements.txt

@@ -0,0 +1,8 @@
+# The order of packages is significant, because pip processes them in the order
+# of appearance. Changing the order has an impact on the overall integration
+# process, which may cause wedges in the gate later.
+
+cotyledon>=1.5.0 # Apache-2.0
+Flask!=0.11,>=0.12.3 # BSD
+kuryr-lib>=0.5.0 # Apache-2.0
+cryptography==2.3.0