From e4bc8a3b1b5b6900de54e9659f3a9d04a9a78595 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?C=C3=A9dric=20Fabianski?= Date: Thu, 31 Oct 2024 11:39:40 +0100 Subject: [PATCH] Add string values --- .tool-versions | 1 - pkg/detectors/openapi/v3yaml/v3yml.go | 20 +++++--- pkg/detectors/typescript/typescript.go | 4 +- pkg/report/output/output.go | 2 +- pkg/report/writer/detectors.go | 2 +- .../detectors/customrule/customrule.go | 1 + .../detectors/customrule/filters/filters.go | 51 +++++++++++-------- .../customrule/filters/filters_test.go | 50 +++++++++--------- .../detectors/customrule/types/types.go | 1 + pkg/scanner/scanner.go | 10 +++- 10 files changed, 84 insertions(+), 58 deletions(-) delete mode 100644 .tool-versions diff --git a/.tool-versions b/.tool-versions deleted file mode 100644 index e4ab54ccd..000000000 --- a/.tool-versions +++ /dev/null @@ -1 +0,0 @@ -golang 1.21.1 diff --git a/pkg/detectors/openapi/v3yaml/v3yml.go b/pkg/detectors/openapi/v3yaml/v3yml.go index 903fbd6ce..368fa2b08 100644 --- a/pkg/detectors/openapi/v3yaml/v3yml.go +++ b/pkg/detectors/openapi/v3yaml/v3yml.go @@ -7,7 +7,6 @@ import ( "github.com/bearer/bearer/pkg/parser" "github.com/bearer/bearer/pkg/parser/nodeid" reporttypes "github.com/bearer/bearer/pkg/report" - "github.com/bearer/bearer/pkg/report/operations/operationshelper" "github.com/bearer/bearer/pkg/report/schema/schemahelper" "github.com/bearer/bearer/pkg/util/file" "github.com/smacker/go-tree-sitter/yaml" @@ -60,11 +59,20 @@ func ProcessFile(idGenerator nodeid.Generator, file *file.FileInfo, report repor return false, err } - foundPaths := make(map[parser.Node]*operationshelper.Operation) - err = yamlparser.AnnotatePaths(tree, foundPaths) - if err != nil { - return false, err - } + // foundPaths := make(map[parser.Node]*operationshelper.Operation) + // err = yamlparser.AnnotatePaths(tree, foundPaths) + // if err != nil { + // return false, err + // } + + // fmt.Printf("FOUND PATHS %#v") + // for i, path := range foundPaths { + // fmt.Printf("\nContent%s", i.Content()) + // fmt.Printf("\nSource %#v", path.Source) + // fmt.Printf("\nPath %s", path.Value.Path) + // fmt.Printf("\nType %s", path.Value.Type) + // fmt.Printf("\nURLs %s", path.Value.Urls) + // } reportadder.AddSchema(file, report, foundSchemas, idGenerator) diff --git a/pkg/detectors/typescript/typescript.go b/pkg/detectors/typescript/typescript.go index ddca9350d..92084484f 100644 --- a/pkg/detectors/typescript/typescript.go +++ b/pkg/detectors/typescript/typescript.go @@ -30,8 +30,8 @@ var ( index: (string) @key) @node (variable_declarator - name: (object_pattern (shorthand_property_identifier_pattern) @key @node) - value: (member_expression) @object) + name: (_) @key @node + value: (_) @object) `) ) diff --git a/pkg/report/output/output.go b/pkg/report/output/output.go index 199dc423f..e35d00fa7 100644 --- a/pkg/report/output/output.go +++ b/pkg/report/output/output.go @@ -49,7 +49,7 @@ func GetData( } // add dataflow to data - if err = GetDataflow(data, report, config, config.Report.Report != flag.ReportDataFlow); err != nil { + if err = GetDataflow(data, report, config, true); err != nil { return data, err } diff --git a/pkg/report/writer/detectors.go b/pkg/report/writer/detectors.go index 9f72b2326..67ba3abba 100644 --- a/pkg/report/writer/detectors.go +++ b/pkg/report/writer/detectors.go @@ -3,10 +3,10 @@ package writer import ( "fmt" "io" - "log" classification "github.com/bearer/bearer/pkg/classification" classificationschema "github.com/bearer/bearer/pkg/classification/schema" + "github.com/rs/zerolog/log" zerolog "github.com/rs/zerolog/log" "github.com/bearer/bearer/pkg/parser" diff --git a/pkg/scanner/detectors/customrule/customrule.go b/pkg/scanner/detectors/customrule/customrule.go index 206d3ec90..3299fdfba 100644 --- a/pkg/scanner/detectors/customrule/customrule.go +++ b/pkg/scanner/detectors/customrule/customrule.go @@ -108,6 +108,7 @@ func (detector *Detector) DetectAt( Pattern: pattern.Pattern, Datatypes: match.DatatypeDetections(), Variables: match.Variables(), + Value: match.Value(), }) } diff --git a/pkg/scanner/detectors/customrule/filters/filters.go b/pkg/scanner/detectors/customrule/filters/filters.go index 2dc9b1a8a..0a932449d 100644 --- a/pkg/scanner/detectors/customrule/filters/filters.go +++ b/pkg/scanner/detectors/customrule/filters/filters.go @@ -29,10 +29,11 @@ func NewResult(matches ...Match) *Result { type Match struct { variables variableshape.Values datatypeDetections []*detectortypes.Detection + value *string } -func NewMatch(variables variableshape.Values, datatypeDetections []*detectortypes.Detection) Match { - return Match{variables: variables, datatypeDetections: datatypeDetections} +func NewMatch(variables variableshape.Values, valueStr *string, datatypeDetections []*detectortypes.Detection) Match { + return Match{variables: variables, value: valueStr, datatypeDetections: datatypeDetections} } func (result *Result) Matches() []Match { @@ -43,6 +44,10 @@ func (match *Match) Variables() variableshape.Values { return match.variables } +func (match *Match) Value() *string { + return match.value +} + func (match *Match) DatatypeDetections() []*detectortypes.Detection { return match.datatypeDetections } @@ -78,7 +83,7 @@ func (filter *Not) Evaluate( log.Trace().Msgf("filters.Not: %t", result) } - return boolResult(patternVariables, result), nil + return boolResult(patternVariables, result, nil), nil } type Either struct { @@ -125,7 +130,7 @@ func (filter *All) Evaluate( if len(filter.Children) == 0 { log.Trace().Msg("filters.All: true (no children)") - return boolResult(patternVariables, true), nil + return boolResult(patternVariables, true, nil), nil } for i, child := range filter.Children { @@ -164,6 +169,7 @@ func (filter *All) joinMatches(matches, childMatches []Match) []Match { if variables, variablesMatch := match.variables.Merge(childMatch.variables); variablesMatch { result = append(result, NewMatch( variables, + nil, // FIXME: this seems like it will create unnecessary duplicates append(match.datatypeDetections, childMatch.datatypeDetections...), )) @@ -182,7 +188,7 @@ func (filter *FilenameRegex) Evaluate( detectorContext detectortypes.Context, patternVariables variableshape.Values, ) (*Result, error) { - return boolResult(patternVariables, filter.Regex.MatchString(detectorContext.Filename())), nil + return boolResult(patternVariables, filter.Regex.MatchString(detectorContext.Filename()), nil), nil } type ImportedVariable struct { @@ -218,7 +224,7 @@ func (filter *Rule) Evaluate( if filter.IsDatatypeRule { log.Trace().Msg("filters.Rule: match (datatype)") - return NewResult(NewMatch(patternVariables, detections)), nil + return NewResult(NewMatch(patternVariables, nil, detections)), nil } if log.Trace().Enabled() { @@ -271,7 +277,7 @@ func (filter *Rule) Evaluate( for _, detectionMatch := range subResult.matches { if variables, variablesMatch := filter.importVariables(patternVariables, detectionMatch.variables); variablesMatch { matched = true - matches = append(matches, NewMatch(variables, detectionMatch.datatypeDetections)) + matches = append(matches, NewMatch(variables, nil, detectionMatch.datatypeDetections)) } } @@ -288,7 +294,7 @@ func (filter *Rule) Evaluate( } if hasPatternVariableMatch { - matches = append(matches, NewMatch(patternVariables, datatypeDetections)) + matches = append(matches, NewMatch(patternVariables, nil, datatypeDetections)) } return NewResult(matches...), nil @@ -329,7 +335,7 @@ func (filter *Values) Evaluate( patternVariables variableshape.Values, ) (*Result, error) { node := patternVariables.Node(filter.Variable) - return boolResult(patternVariables, slices.Contains(filter.Values, node.Content())), nil + return boolResult(patternVariables, slices.Contains(filter.Values, node.Content()), nil), nil } type Regex struct { @@ -354,7 +360,7 @@ func (filter *Regex) Evaluate( ) } - return boolResult(patternVariables, result), nil + return boolResult(patternVariables, result, nil), nil } type StringLengthLessThan struct { @@ -372,7 +378,7 @@ func (filter *StringLengthLessThan) Evaluate( return nil, err } - return boolResult(patternVariables, len(value) < filter.Value), nil + return boolResult(patternVariables, len(value) < filter.Value, nil), nil } type StringRegex struct { @@ -409,7 +415,10 @@ func (filter *StringRegex) Evaluate( ) } - return boolResult(patternVariables, result), nil + // bar, _ := url.PrepareURLValue(value) + // log.Debug().Msgf("filters %s -> %s", value, bar) + + return boolResult(patternVariables, result, &value), nil } type EntropyGreaterThan struct { @@ -448,7 +457,7 @@ func (filter *EntropyGreaterThan) Evaluate( ) } - return boolResult(patternVariables, result), nil + return boolResult(patternVariables, result, nil), nil } type IntegerLessThan struct { @@ -466,7 +475,7 @@ func (filter *IntegerLessThan) Evaluate( return nil, err } - return boolResult(patternVariables, value < filter.Value), nil + return boolResult(patternVariables, value < filter.Value, nil), nil } type IntegerLessThanOrEqual struct { @@ -484,7 +493,7 @@ func (filter *IntegerLessThanOrEqual) Evaluate( return nil, err } - return boolResult(patternVariables, value <= filter.Value), nil + return boolResult(patternVariables, value <= filter.Value, nil), nil } type IntegerGreaterThan struct { @@ -502,7 +511,7 @@ func (filter *IntegerGreaterThan) Evaluate( return nil, err } - return boolResult(patternVariables, value > filter.Value), nil + return boolResult(patternVariables, value > filter.Value, nil), nil } type IntegerGreaterThanOrEqual struct { @@ -520,7 +529,7 @@ func (filter *IntegerGreaterThanOrEqual) Evaluate( return nil, err } - return boolResult(patternVariables, value >= filter.Value), nil + return boolResult(patternVariables, value >= filter.Value, nil), nil } type Unknown struct{} @@ -553,13 +562,13 @@ func parseInteger(node *tree.Node) (int, bool, error) { return value, true, nil } -func boolResult(patternVariables variableshape.Values, value bool) *Result { - return NewResult(boolMatches(patternVariables, value)...) +func boolResult(patternVariables variableshape.Values, value bool, valueStr *string) *Result { + return NewResult(boolMatches(patternVariables, value, valueStr)...) } -func boolMatches(patternVariables variableshape.Values, value bool) []Match { +func boolMatches(patternVariables variableshape.Values, value bool, valueStr *string) []Match { if value { - return []Match{NewMatch(patternVariables, nil)} + return []Match{NewMatch(patternVariables, valueStr, nil)} } else { return nil } diff --git a/pkg/scanner/detectors/customrule/filters/filters_test.go b/pkg/scanner/detectors/customrule/filters/filters_test.go index b9ace75cf..8d1c1cf21 100644 --- a/pkg/scanner/detectors/customrule/filters/filters_test.go +++ b/pkg/scanner/detectors/customrule/filters/filters_test.go @@ -69,7 +69,7 @@ var _ = Describe("Not", func() { When("the child filter has a match", func() { BeforeEach(func(ctx SpecContext) { filter = &filters.Not{ - Child: &MockFilter{result: filters.NewResult(filters.NewMatch(nil, nil))}, + Child: &MockFilter{result: filters.NewResult(filters.NewMatch(nil, nil, nil))}, } }) @@ -87,7 +87,7 @@ var _ = Describe("Not", func() { It("returns a result with a match using the pattern variables", func(ctx SpecContext) { Expect(filter.Evaluate(defaultDetectorContext, patternVariables)).To(Equal( - filters.NewResult(filters.NewMatch(patternVariables, nil)), + filters.NewResult(filters.NewMatch(patternVariables, nil, nil)), )) }) }) @@ -110,9 +110,9 @@ var _ = Describe("Either", func() { patternVariables := []*tree.Node{{ID: 42}} When("there are child filter matches", func() { - match1 := filters.NewMatch([]*tree.Node{{ID: 1}}, nil) - match2 := filters.NewMatch([]*tree.Node{{ID: 2}}, nil) - match3 := filters.NewMatch([]*tree.Node{{ID: 3}}, nil) + match1 := filters.NewMatch([]*tree.Node{{ID: 1}}, nil, nil) + match2 := filters.NewMatch([]*tree.Node{{ID: 2}}, nil, nil) + match3 := filters.NewMatch([]*tree.Node{{ID: 3}}, nil, nil) BeforeEach(func(ctx SpecContext) { filter = &filters.Either{ @@ -193,12 +193,12 @@ var _ = Describe("All", func() { nodes = parseNodes(ctx, []string{"n1", "n2", "n3", "n4", "n5", "n6", "n7", "n8"}) patternVariables = []*tree.Node{nodes[0], nil, nil, nil} - match1 = filters.NewMatch([]*tree.Node{nodes[0], nil, nil, nil}, []*detectortypes.Detection{datatype1}) - match2 = filters.NewMatch([]*tree.Node{nil, nodes[2], nodes[4], nil}, []*detectortypes.Detection{datatype2}) - match3 = filters.NewMatch([]*tree.Node{nil, nodes[3], nodes[5], nil}, []*detectortypes.Detection{datatype3}) - match4 = filters.NewMatch([]*tree.Node{nodes[0], nodes[3], nil, nodes[6]}, []*detectortypes.Detection{datatype4}) - match5 = filters.NewMatch([]*tree.Node{nodes[0], nodes[3], nil, nodes[7]}, []*detectortypes.Detection{datatype5}) - discordantMatch = filters.NewMatch([]*tree.Node{nodes[1], nil, nil, nil}, []*detectortypes.Detection{discordantDatatype}) + match1 = filters.NewMatch([]*tree.Node{nodes[0], nil, nil, nil}, nil, []*detectortypes.Detection{datatype1}) + match2 = filters.NewMatch([]*tree.Node{nil, nodes[2], nodes[4], nil}, nil, []*detectortypes.Detection{datatype2}) + match3 = filters.NewMatch([]*tree.Node{nil, nodes[3], nodes[5], nil}, nil, []*detectortypes.Detection{datatype3}) + match4 = filters.NewMatch([]*tree.Node{nodes[0], nodes[3], nil, nodes[6]}, nil, []*detectortypes.Detection{datatype4}) + match5 = filters.NewMatch([]*tree.Node{nodes[0], nodes[3], nil, nodes[7]}, nil, []*detectortypes.Detection{datatype5}) + discordantMatch = filters.NewMatch([]*tree.Node{nodes[1], nil, nil, nil}, nil, []*detectortypes.Detection{discordantDatatype}) }) When("there is a single child filter with matches", func() { @@ -238,10 +238,12 @@ var _ = Describe("All", func() { Expect(result.Matches()).To(ContainElements( filters.NewMatch( []*tree.Node{nodes[0], nodes[3], nodes[5], nodes[6]}, + nil, []*detectortypes.Detection{datatype1, datatype3, datatype4}, ), filters.NewMatch( []*tree.Node{nodes[0], nodes[3], nodes[5], nodes[7]}, + nil, []*detectortypes.Detection{datatype1, datatype3, datatype5}, ), )) @@ -289,7 +291,7 @@ var _ = Describe("All", func() { It("returns a result with a single match using the pattern variables", func(ctx SpecContext) { Expect(filter.Evaluate(defaultDetectorContext, patternVariables)).To(Equal( - filters.NewResult(filters.NewMatch(patternVariables, nil)), + filters.NewResult(filters.NewMatch(patternVariables, nil, nil)), )) }) }) @@ -306,7 +308,7 @@ var _ = Describe("FilenameRegex", func() { It("returns a result with a match using the pattern variables", func(ctx SpecContext) { Expect(filter.Evaluate(defaultDetectorContext, patternVariables)).To(Equal( - filters.NewResult(filters.NewMatch(patternVariables, nil)), + filters.NewResult(filters.NewMatch(patternVariables, nil, nil)), )) }) }) @@ -341,7 +343,7 @@ var _ = Describe("Values", func() { It("returns a result with a match using the pattern variables", func(ctx SpecContext) { Expect(filter.Evaluate(defaultDetectorContext, patternVariables)).To(Equal( - filters.NewResult(filters.NewMatch(patternVariables, nil)), + filters.NewResult(filters.NewMatch(patternVariables, nil, nil)), )) }) }) @@ -373,7 +375,7 @@ var _ = Describe("Regex", func() { It("returns a result with a match using the pattern variables", func(ctx SpecContext) { Expect(filter.Evaluate(defaultDetectorContext, patternVariables)).To(Equal( - filters.NewResult(filters.NewMatch(patternVariables, nil)), + filters.NewResult(filters.NewMatch(patternVariables, nil, nil)), )) }) }) @@ -407,7 +409,7 @@ var _ = Describe("StringLengthLessThan", func() { It("returns a result with a match using the pattern variables", func(ctx SpecContext) { Expect(filter.Evaluate(detectorContext, patternVariables)).To(Equal( - filters.NewResult(filters.NewMatch(patternVariables, nil)), + filters.NewResult(filters.NewMatch(patternVariables, nil, nil)), )) }) }) @@ -452,7 +454,7 @@ var _ = Describe("StringRegex", func() { It("returns a result with a match using the pattern variables", func(ctx SpecContext) { Expect(filter.Evaluate(detectorContext, patternVariables)).To(Equal( - filters.NewResult(filters.NewMatch(patternVariables, nil)), + filters.NewResult(filters.NewMatch(patternVariables, nil, nil)), )) }) }) @@ -498,7 +500,7 @@ var _ = Describe("EntropyGreaterThan", func() { It("returns a result with a match using the pattern variables", func(ctx SpecContext) { Expect(filter.Evaluate(detectorContext, patternVariables)).To(Equal( - filters.NewResult(filters.NewMatch(patternVariables, nil)), + filters.NewResult(filters.NewMatch(patternVariables, nil, nil)), )) }) }) @@ -541,7 +543,7 @@ var _ = Describe("IntegerLessThan", func() { It("returns a result with a match using the pattern variables", func(ctx SpecContext) { Expect(filter.Evaluate(defaultDetectorContext, patternVariables)).To(Equal( - filters.NewResult(filters.NewMatch(patternVariables, nil)), + filters.NewResult(filters.NewMatch(patternVariables, nil, nil)), )) }) }) @@ -584,7 +586,7 @@ var _ = Describe("IntegerLessThanOrEqual", func() { It("returns a result with a match using the pattern variables", func(ctx SpecContext) { Expect(filter.Evaluate(defaultDetectorContext, patternVariables)).To(Equal( - filters.NewResult(filters.NewMatch(patternVariables, nil)), + filters.NewResult(filters.NewMatch(patternVariables, nil, nil)), )) }) @@ -596,7 +598,7 @@ var _ = Describe("IntegerLessThanOrEqual", func() { It("returns a result with a match using the pattern variables", func(ctx SpecContext) { Expect(filter.Evaluate(defaultDetectorContext, patternVariables)).To(Equal( - filters.NewResult(filters.NewMatch(patternVariables, nil)), + filters.NewResult(filters.NewMatch(patternVariables, nil, nil)), )) }) }) @@ -639,7 +641,7 @@ var _ = Describe("IntegerGreaterThan", func() { It("returns a result with a match using the pattern variables", func(ctx SpecContext) { Expect(filter.Evaluate(defaultDetectorContext, patternVariables)).To(Equal( - filters.NewResult(filters.NewMatch(patternVariables, nil)), + filters.NewResult(filters.NewMatch(patternVariables, nil, nil)), )) }) }) @@ -682,7 +684,7 @@ var _ = Describe("IntegerGreaterThanOrEqual", func() { It("returns a result with a match using the pattern variables", func(ctx SpecContext) { Expect(filter.Evaluate(defaultDetectorContext, patternVariables)).To(Equal( - filters.NewResult(filters.NewMatch(patternVariables, nil)), + filters.NewResult(filters.NewMatch(patternVariables, nil, nil)), )) }) }) @@ -694,7 +696,7 @@ var _ = Describe("IntegerGreaterThanOrEqual", func() { It("returns a result with a match using the pattern variables", func(ctx SpecContext) { Expect(filter.Evaluate(defaultDetectorContext, patternVariables)).To(Equal( - filters.NewResult(filters.NewMatch(patternVariables, nil)), + filters.NewResult(filters.NewMatch(patternVariables, nil, nil)), )) }) }) diff --git a/pkg/scanner/detectors/customrule/types/types.go b/pkg/scanner/detectors/customrule/types/types.go index 3c6f4e03b..5a0fc77ac 100644 --- a/pkg/scanner/detectors/customrule/types/types.go +++ b/pkg/scanner/detectors/customrule/types/types.go @@ -9,4 +9,5 @@ type Data struct { Pattern string Datatypes []*detectortypes.Detection Variables variableshape.Values + Value *string } diff --git a/pkg/scanner/scanner.go b/pkg/scanner/scanner.go index 56d3c40e0..da0622076 100644 --- a/pkg/scanner/scanner.go +++ b/pkg/scanner/scanner.go @@ -91,6 +91,12 @@ func (scanner *Scanner) Scan( data := detection.Data.(customruletypes.Data) if len(data.Datatypes) == 0 { + var value = "" + + if data.Value != nil { + value = *data.Value + } + report.AddDetection(reportdetections.TypeCustomRisk, detectorType, source.New( @@ -100,14 +106,14 @@ func (scanner *Scanner) Scan( detection.MatchNode.ContentStart.Column, detection.MatchNode.ContentEnd.Line, detection.MatchNode.ContentEnd.Column, - "", + value, ), reportschema.Source{ StartLineNumber: detection.MatchNode.ContentStart.Line, EndLineNumber: detection.MatchNode.ContentEnd.Line, StartColumnNumber: detection.MatchNode.ContentStart.Column, EndColumnNumber: detection.MatchNode.ContentEnd.Column, - Content: "", + Content: value, }) }