From 5e78fe36a8ca54addc54e54fdadef308d8bd4892 Mon Sep 17 00:00:00 2001
From: mcoops <70203682+mcoops@users.noreply.github.com>
Date: Mon, 15 Mar 2021 01:02:01 +1000
Subject: [PATCH] python: add initial (#14)

* python: add initial code to scan requirements.txt file

Signed-off-by: mcoops <cooper.d.mark@gmail.com>
---
 deplist.go                     | 23 +++++++++++++
 deplist_test.go                | 23 +++++++++++--
 internal/scan/python.go        | 59 ++++++++++++++++++++++++++++++++++
 test/testRepo/requirements.txt |  8 +++++
 4 files changed, 111 insertions(+), 2 deletions(-)
 create mode 100755 internal/scan/python.go
 create mode 100644 test/testRepo/requirements.txt

diff --git a/deplist.go b/deplist.go
index 1b91a1a..a321ebf 100644
--- a/deplist.go
+++ b/deplist.go
@@ -52,6 +52,7 @@ func GetDeps(fullPath string) ([]Dependency, Bitmask, error) {
 	pomPath := filepath.Join(fullPath, "pom.xml")
 	goPath := filepath.Join(fullPath, "go.mod")
 	rubyPath := filepath.Join(fullPath, "Gemfile.lock")
+	pythonPath := filepath.Join(fullPath, "requirements.txt")
 
 	// point at the parent repo, but can't assume where the indicators will be
 	err := filepath.Walk(fullPath, func(path string, info os.FileInfo, err error) error {
@@ -87,6 +88,7 @@ func GetDeps(fullPath string) ([]Dependency, Bitmask, error) {
 							DepType: LangNodeJS,
 							Path:    name,
 							Version: strings.Replace(version, "v", "", 1),
+							Files:   []string{},
 						})
 				}
 			}
@@ -128,6 +130,7 @@ func GetDeps(fullPath string) ([]Dependency, Bitmask, error) {
 							DepType: LangJava,
 							Path:    name,
 							Version: strings.Replace(version, "v", "", 1),
+							Files:   []string{},
 						})
 				}
 			case rubyPath:
@@ -146,6 +149,26 @@ func GetDeps(fullPath string) ([]Dependency, Bitmask, error) {
 							DepType: LangRuby,
 							Path:    name,
 							Version: strings.Replace(version, "v", "", 1),
+							Files:   []string{},
+						})
+				}
+			case pythonPath:
+				pkgs, err := scan.GetPythonDeps(path)
+				if err != nil {
+					return err
+				}
+
+				if len(pkgs) > 0 {
+					foundTypes.DepFoundAddFlag(LangPython)
+				}
+
+				for name, version := range pkgs {
+					deps = append(deps,
+						Dependency{
+							DepType: LangPython,
+							Path:    name,
+							Version: version,
+							Files:   []string{},
 						})
 				}
 			}
diff --git a/deplist_test.go b/deplist_test.go
index 36969ff..8f686b3 100644
--- a/deplist_test.go
+++ b/deplist_test.go
@@ -183,6 +183,13 @@ func BuildWant() []Dependency {
 		"addressable",
 	}
 
+	pythonSet := []string{
+		"cotyledon",
+		"Flask",
+		"kuryr-lib",
+		"cryptography",
+	}
+
 	for _, n := range golangPaths {
 		d := Dependency{
 			DepType: 1,
@@ -219,6 +226,18 @@ func BuildWant() []Dependency {
 		}
 		deps = append(deps, d)
 	}
+
+	for _, n := range pythonSet {
+		d := Dependency{
+			DepType: LangPython,
+			Path:    n,
+		}
+		deps = append(deps, d)
+	}
+
+	end = len(deps) - 1 // get the cryptography ver
+	deps[end].Version = "2.3.0"
+
 	return deps
 }
 
@@ -227,8 +246,8 @@ func TestGetDeps(t *testing.T) {
 
 	got, gotBitmask, _ := GetDeps("test/testRepo")
 
-	if gotBitmask != 23 {
-		t.Errorf("GotBitmask() != 7; got: %d", gotBitmask)
+	if gotBitmask != 31 {
+		t.Errorf("GotBitmask() != 31; got: %d", gotBitmask)
 	}
 
 	// iterate thru and compare
diff --git a/internal/scan/python.go b/internal/scan/python.go
new file mode 100755
index 0000000..2d323d7
--- /dev/null
+++ b/internal/scan/python.go
@@ -0,0 +1,59 @@
+package scan
+
+import (
+	"bufio"
+	"os"
+	"regexp"
+	"strings"
+)
+
+// Account for >, <, >=, <=, ==, !=, ~= and *
+var /* const */ re = regexp.MustCompile(`[<>!~*]+`)
+
+func max(x, y int) int {
+	if x > y {
+		return x
+	}
+	return y
+}
+
+func GetPythonDeps(path string) (map[string]string, error) {
+	gathered := make(map[string]string)
+
+	file, err := os.Open(path)
+
+	if err != nil {
+		return nil, err
+	}
+
+	defer file.Close()
+
+	scanner := bufio.NewScanner(file)
+
+	for scanner.Scan() {
+		line := scanner.Text()
+
+		// skip comments
+		if strings.HasPrefix(line, "#") || line == "" {
+			continue
+		}
+
+		// easy case, elasticsearch-curator==5.8.1
+		// record name and version, only for ==
+		idx := strings.LastIndex(line, "==")
+		if idx > 0 {
+			gathered[line[:idx]] = line[idx+2:]
+			continue
+		}
+
+		// every other permitation just use the name as we can't guarantee
+		// the version, just grab the name using first occurance
+		match := re.FindStringIndex(line)
+
+		if match != nil {
+			gathered[line[:match[0]]] = ""
+		}
+	}
+
+	return gathered, nil
+}
diff --git a/test/testRepo/requirements.txt b/test/testRepo/requirements.txt
new file mode 100644
index 0000000..d116f28
--- /dev/null
+++ b/test/testRepo/requirements.txt
@@ -0,0 +1,8 @@
+# The order of packages is significant, because pip processes them in the order
+# of appearance. Changing the order has an impact on the overall integration
+# process, which may cause wedges in the gate later.
+
+cotyledon>=1.5.0 # Apache-2.0
+Flask!=0.11,>=0.12.3 # BSD
+kuryr-lib>=0.5.0 # Apache-2.0
+cryptography==2.3.0