bring back openapi to dataflow

.github/workflows/lint.yml

@@ -7,7 +7,7 @@ on:
       - synchronize
       - reopened
     paths-ignore:
-      - 'docs/**'
+      - "docs/**"
 
 permissions:
   # Required: allow read access to the content for analysis.
@@ -16,7 +16,7 @@ permissions:
   pull-requests: read
   # Optional: Allow write access to checks to allow the action to annotate code in the PR.
   checks: write
-  
+
 jobs:
   golangci:
     name: lint

pkg/detectors/internal/testhelper/testhelper.go

@@ -16,6 +16,7 @@ import (
 	reportdetectors "github.com/bearer/bearer/pkg/report/detectors"
 	"github.com/bearer/bearer/pkg/report/frameworks"
 	"github.com/bearer/bearer/pkg/report/interfaces"
+	"github.com/bearer/bearer/pkg/report/operations"
 	"github.com/bearer/bearer/pkg/report/schema"
 	"github.com/bearer/bearer/pkg/report/schema/datatype"
 	"github.com/bearer/bearer/pkg/report/secret"
@@ -211,3 +212,16 @@ func (report *InMemoryReport) AddError(filePath string, err error) {
 		File:    filePath,
 	})
 }
+
+func (report *InMemoryReport) AddOperation(
+	detectorType reportdetectors.Type,
+	operation operations.Operation,
+	source source.Source,
+) {
+	report.Detections = append(report.Detections, &detections.Detection{
+		DetectorType: detectorType,
+		Value:        operation,
+		Source:       source,
+		Type:         detections.TypeOperation,
+	})
+}

pkg/detectors/openapi/reportadder/reportadder.go

@@ -1,18 +1,24 @@
 package reportadder
 
 import (
+	"regexp"
 	"sort"
+	"strings"
 
 	"github.com/bearer/bearer/pkg/parser"
 	"github.com/bearer/bearer/pkg/parser/nodeid"
 	reporttypes "github.com/bearer/bearer/pkg/report"
 	"github.com/bearer/bearer/pkg/report/detectors"
+	"github.com/bearer/bearer/pkg/report/operations"
+	"github.com/bearer/bearer/pkg/report/operations/operationshelper"
 	"github.com/bearer/bearer/pkg/report/schema"
 	"github.com/bearer/bearer/pkg/report/schema/schemahelper"
 	"github.com/bearer/bearer/pkg/util/file"
 	"github.com/bearer/bearer/pkg/util/stringutil"
 )
 
+var regexpPathVariable = regexp.MustCompile(`\{.+\}`)
+
 type SortableSchema struct {
 	Key   parser.Node
 	Value *schemahelper.Schema
@@ -62,7 +68,7 @@ func AddSchema(file *file.FileInfo, report reporttypes.Report, foundValues map[p
 		if !report.SchemaGroupIsOpen() {
 			report.SchemaGroupBegin(
 				detectors.DetectorOpenAPI,
-				nil,
+				&node,
 				schema.Value,
 				&schema.Source,
 				nil,
@@ -94,3 +100,41 @@ func convertSchema(value string) string {
 		return schema.SimpleTypeObject
 	}
 }
+
+func AddOperations(file *file.FileInfo, report reporttypes.Report, foundValues map[parser.Node]*operationshelper.Operation, servers []operations.Url) {
+	// we need sorted schemas so our reports are consistent and repeatable
+	var sortedOperations []*operationshelper.Operation
+	for _, operation := range foundValues {
+		sortedOperations = append(sortedOperations, operation)
+	}
+	sort.Slice(sortedOperations, func(i, j int) bool {
+		lineNumberA := sortedOperations[i].Source.StartLineNumber
+		lineNumberB := sortedOperations[j].Source.StartLineNumber
+		return *lineNumberA < *lineNumberB
+	})
+	for _, operation := range sortedOperations {
+		operation.Source.Language = file.Language
+		operation.Source.LanguageType = file.LanguageTypeString()
+		operation.Value.Path = standardizeOperationPath(stringutil.StripQuotes(operation.Value.Path))
+		operation.Value.Type = standardizeOperationType(stringutil.StripQuotes(operation.Value.Type))
+		operation.Value.Urls = servers
+		report.AddOperation(detectors.DetectorOpenAPI, operation.Value, operation.Source)
+	}
+}
+
+func standardizeOperationType(input string) (output string) {
+	input = strings.ToUpper(input)
+	supportedvalues := []string{operations.TypeGet, operations.TypeDelete, operations.TypePost, operations.TypePut}
+
+	for _, v := range supportedvalues {
+		if input == v {
+			return v
+		}
+	}
+
+	return operations.TypeOther
+}
+
+func standardizeOperationPath(input string) (output string) {
+	return regexpPathVariable.ReplaceAllString(input, "*")
+}

pkg/detectors/openapi/v2json/server.go

@@ -0,0 +1,87 @@
+package v2json
+
+import (
+	"strings"
+
+	"github.com/bearer/bearer/pkg/parser"
+	"github.com/bearer/bearer/pkg/report/operations"
+	"github.com/bearer/bearer/pkg/util/stringutil"
+	"github.com/smacker/go-tree-sitter/javascript"
+)
+
+var queryServersHost = parser.QueryMustCompile(javascript.GetLanguage(), `
+(expression_statement
+	(object
+		(pair
+			key: (string) @helper_host
+			(#match? @helper_host "^\"host\"$")
+			value: (string) @param_host
+		)
+	)
+)
+`)
+
+var queryServersBasePath = parser.QueryMustCompile(javascript.GetLanguage(), `
+(expression_statement
+	(object
+		(pair
+			key: (string) @helper_basePath
+			(#match? @helper_basePath "^\"basePath\"$")
+			value: (string) @param_base_path
+		)
+	)
+)
+`)
+
+var queryServerSchemes = parser.QueryMustCompile(javascript.GetLanguage(), `
+(expression_statement
+	(object
+		(pair
+			key: (string) @helper_schemes
+			(#match? @helper_schemes "^\"schemes\"$")
+			value: (array
+				(string) @param_schemes
+			)
+		)
+	)
+)
+`)
+
+func findServers(tree *parser.Tree) (result []operations.Url) {
+	host := ""
+	captures := tree.QueryConventional(queryServersHost)
+	for _, capture := range captures {
+		host = stringutil.StripQuotes(capture["param_host"].Content())
+	}
+
+	basePath := ""
+	captures = tree.QueryConventional(queryServersBasePath)
+	for _, capture := range captures {
+		basePath = stringutil.StripQuotes(capture["param_base_path"].Content())
+	}
+
+	schemes := make([]string, 0)
+	captures = tree.QueryConventional(queryServerSchemes)
+	for _, capture := range captures {
+		schemes = append(schemes, stringutil.StripQuotes(capture["param_schemes"].Content()))
+	}
+
+	if len(schemes) == 0 {
+		schemes = append(schemes, "http://", "https://")
+	}
+
+	for _, scheme := range schemes {
+		if !strings.HasSuffix(scheme, "://") {
+			scheme = scheme + "://"
+		}
+
+		if !strings.HasPrefix(basePath, "/") {
+			basePath = "/" + basePath
+		}
+		result = append(result, operations.Url{
+			Url: scheme + host + basePath,
+		})
+	}
+
+	return result
+}

pkg/detectors/openapi/v2json/v2json.go

@@ -68,6 +68,8 @@ func ProcessFile(idGenerator nodeid.Generator, file *file.FileInfo, report repor
 		return false, err
 	}
 
+	servers := findServers(tree)
+	reportadder.AddOperations(file, report, foundPaths, servers)
 	reportadder.AddSchema(file, report, foundSchemas, idGenerator)
 
 	return true, err

pkg/detectors/openapi/v2yaml/server.go

@@ -0,0 +1,95 @@
+package v2yaml
+
+import (
+	"strings"
+
+	"github.com/bearer/bearer/pkg/parser"
+	"github.com/bearer/bearer/pkg/report/operations"
+	"github.com/bearer/bearer/pkg/util/stringutil"
+	"github.com/smacker/go-tree-sitter/yaml"
+)
+
+var queryServersHost = parser.QueryMustCompile(yaml.GetLanguage(), `
+(document
+	(block_node
+    	(block_mapping
+        	(block_mapping_pair
+              key: (flow_node) @helper_host
+			  value: (flow_node) @param_host
+            )
+        )
+    )
+)
+`)
+
+var queryServersBasePath = parser.QueryMustCompile(yaml.GetLanguage(), `
+(document
+	(block_node
+    	(block_mapping
+        	(block_mapping_pair
+              key: (flow_node) @helper_basePath
+			  value: (flow_node) @param_base_path
+            )
+        )
+    )
+)
+`)
+
+var queryServerSchemes = parser.QueryMustCompile(yaml.GetLanguage(), `
+(document
+	(block_node
+    	(block_mapping
+        	(block_mapping_pair
+              key: (flow_node) @helper_schemes
+              value:
+              	(block_node
+                	(block_sequence
+                    	(block_sequence_item
+                        	(flow_node) @param_scheme
+                        )
+                    )
+                )
+            )
+        )
+    )
+)
+`)
+
+func findServers(tree *parser.Tree) (result []operations.Url) {
+	host := ""
+	captures := tree.QueryConventional(queryServersHost)
+	for _, capture := range captures {
+		host = stringutil.StripQuotes(capture["param_host"].Content())
+	}
+
+	basePath := ""
+	captures = tree.QueryConventional(queryServersBasePath)
+	for _, capture := range captures {
+		basePath = stringutil.StripQuotes(capture["param_base_path"].Content())
+	}
+
+	schemes := make([]string, 0)
+	captures = tree.QueryConventional(queryServerSchemes)
+	for _, capture := range captures {
+		schemes = append(schemes, stringutil.StripQuotes(capture["param_scheme"].Content()))
+	}
+
+	if len(schemes) == 0 {
+		schemes = append(schemes, "http://", "https://")
+	}
+
+	for _, scheme := range schemes {
+		if !strings.HasSuffix(scheme, "://") {
+			scheme = scheme + "://"
+		}
+
+		if !strings.HasPrefix(basePath, "/") {
+			basePath = "/" + basePath
+		}
+		result = append(result, operations.Url{
+			Url: scheme + host + basePath,
+		})
+	}
+
+	return result
+}

pkg/detectors/openapi/v2yaml/v2yml.go

@@ -68,6 +68,8 @@ func ProcessFile(idGenerator nodeid.Generator, file *file.FileInfo, report repor
 		return false, err
 	}
 
+	servers := findServers(tree)
+	reportadder.AddOperations(file, report, foundPaths, servers)
 	reportadder.AddSchema(file, report, foundValues, idGenerator)
 
 	return true, err

pkg/detectors/openapi/v3json/v3json.go

@@ -67,6 +67,9 @@ func ProcessFile(idGenerator nodeid.Generator, file *file.FileInfo, report repor
 		return false, err
 	}
 
+	servers := queries.FindUrls(file)
+
+	reportadder.AddOperations(file, report, foundPaths, servers)
 	reportadder.AddSchema(file, report, foundSchemas, idGenerator)
 
 	return true, err

pkg/detectors/openapi/v3yaml/v3yml.go

@@ -66,6 +66,8 @@ func ProcessFile(idGenerator nodeid.Generator, file *file.FileInfo, report repor
 		return false, err
 	}
 
+	servers := queries.FindUrls(file)
+	reportadder.AddOperations(file, report, foundPaths, servers)
 	reportadder.AddSchema(file, report, foundSchemas, idGenerator)
 
 	return true, err

pkg/report/detections/detections.go

@@ -28,6 +28,7 @@ var TypeCustom DetectionType = "custom"
 var TypeCustomClassified DetectionType = "custom_classified"
 var TypeCustomRisk DetectionType = "custom_risk"
 var TypeExpectedDetection DetectionType = "expected_detection"
+var TypeOperation DetectionType = "operation"
 
 type ReportDetection interface {
 	AddDetection(detectionType DetectionType, detectorType detectors.Type, source source.Source, value interface{})