From 898d80fdcb9a32e44226e39e2a107451e6257b68 Mon Sep 17 00:00:00 2001 From: Michael Pivonka Date: Thu, 23 May 2024 10:01:35 -0500 Subject: [PATCH 1/4] Exposed rule's metadata in golang package --- go/main.go | 32 +++++++++++++++++++++++++++++++- go/scanner_test.go | 21 ++++++++++++++------- 2 files changed, 45 insertions(+), 8 deletions(-) diff --git a/go/main.go b/go/main.go index 0ca73e422..1d29e7b4a 100644 --- a/go/main.go +++ b/go/main.go @@ -6,12 +6,12 @@ package yara_x // #include import "C" import ( + "encoding/json" "errors" "runtime" "unsafe" ) - // Compile receives YARA source code and returns compiled [Rules] that can be // used for scanning data. func Compile(src string, opts ...CompileOption) (*Rules, error) { @@ -88,6 +88,7 @@ type Rule struct { identifier string cPatterns *C.YRX_PATTERNS patterns []Pattern + metadata []Metadata } // Creates a new Rule from it's C counterpart. @@ -107,11 +108,29 @@ func newRule(cRule *C.YRX_RULE) *Rule { identifier := C.GoStringN((*C.char)(unsafe.Pointer(str)), C.int(len)) + var metadata []Metadata + + var buf *C.YRX_BUFFER + if C.yrx_rule_metadata_as_json(cRule, &buf) == C.SUCCESS { + var tmp [][]interface{} + if err := json.Unmarshal(C.GoBytes(unsafe.Pointer(buf.data), C.int(buf.length)), &tmp); err != nil { + panic("yrx_rule_metadata_as_json") + } + for _, v := range tmp { + metadata = append(metadata, Metadata{ + Identifier: v[0].(string), + Value: v[1], + }) + } + defer C.yrx_buffer_destroy(buf) + } + rule := &Rule{ namespace, identifier, C.yrx_rule_patterns(cRule), nil, + metadata, } runtime.SetFinalizer(rule, (*Rule).destroy) @@ -133,6 +152,11 @@ func (r *Rule) Namespace() string { return r.namespace } +// Metadata returns the rule's metadata +func (r *Rule) Metadata() []Metadata { + return r.metadata +} + // Patterns returns the patterns defined by this rule. func (r *Rule) Patterns() []Pattern { // If this method was called before, return the patterns already cached. @@ -165,6 +189,12 @@ func (r *Rule) Patterns() []Pattern { return r.patterns } +// Metadata represents a metadata in a Rule. +type Metadata struct { + Identifier string + Value interface{} +} + // Pattern represents a pattern in a Rule. type Pattern struct { identifier string diff --git a/go/scanner_test.go b/go/scanner_test.go index 17f74355d..d92f90ff4 100644 --- a/go/scanner_test.go +++ b/go/scanner_test.go @@ -5,28 +5,35 @@ import ( "runtime" "testing" "time" + + "github.com/stretchr/testify/assert" ) -import "github.com/stretchr/testify/assert" func TestScanner1(t *testing.T) { - r, _ := Compile("rule t { condition: true }") + r, _ := Compile("rule t { meta: some_int = 1 some_bool = true condition: true }") s := NewScanner(r) - matchingRules, _:= s.Scan([]byte{}) + matchingRules, _ := s.Scan([]byte{}) assert.Len(t, matchingRules, 1) assert.Equal(t, "t", matchingRules[0].Identifier()) assert.Equal(t, "default", matchingRules[0].Namespace()) + assert.Equal(t, "some_int", matchingRules[0].Metadata()[0].Identifier) + assert.Equal(t, float64(1), matchingRules[0].Metadata()[0].Value) + assert.Equal(t, "some_bool", matchingRules[0].Metadata()[1].Identifier) + assert.Equal(t, true, matchingRules[0].Metadata()[1].Value) assert.Len(t, matchingRules[0].Patterns(), 0) } func TestScanner2(t *testing.T) { - r, _ := Compile(`rule t { strings: $bar = "bar" condition: $bar }`) + r, _ := Compile(`rule t { meta: some_string = "hello" strings: $bar = "bar" condition: $bar }`) s := NewScanner(r) matchingRules, _ := s.Scan([]byte("foobar")) assert.Len(t, matchingRules, 1) assert.Equal(t, "t", matchingRules[0].Identifier()) assert.Equal(t, "default", matchingRules[0].Namespace()) + assert.Equal(t, "some_string", matchingRules[0].Metadata()[0].Identifier) + assert.Equal(t, "hello", matchingRules[0].Metadata()[0].Value) assert.Len(t, matchingRules[0].Patterns(), 1) assert.Equal(t, "$bar", matchingRules[0].Patterns()[0].Identifier()) @@ -76,7 +83,7 @@ func TestScanner4(t *testing.T) { func TestScannerTimeout(t *testing.T) { r, _ := Compile("rule t { strings: $a = /a(.*)*a/ condition: $a }") s := NewScanner(r) - s.SetTimeout(1*time.Second) - _, err := s.Scan(bytes.Repeat([]byte("a"), 9000)) + s.SetTimeout(1 * time.Second) + _, err := s.Scan(bytes.Repeat([]byte("a"), 10000)) assert.ErrorIs(t, err, ErrTimeout) -} \ No newline at end of file +} From c11d240aa560cf13d88c72f8961b224ead74dc97 Mon Sep 17 00:00:00 2001 From: Michael Pivonka Date: Fri, 24 May 2024 11:46:27 -0500 Subject: [PATCH 2/4] feat(go) Implemented APIs for obtaining rule's metadata. --- go/main.go | 65 ++++++++++++++++++++++++++++------------------ go/scanner_test.go | 26 +++++++++++++------ 2 files changed, 58 insertions(+), 33 deletions(-) diff --git a/go/main.go b/go/main.go index 1d29e7b4a..26de063d9 100644 --- a/go/main.go +++ b/go/main.go @@ -6,7 +6,6 @@ package yara_x // #include import "C" import ( - "encoding/json" "errors" "runtime" "unsafe" @@ -88,6 +87,7 @@ type Rule struct { identifier string cPatterns *C.YRX_PATTERNS patterns []Pattern + cMetadata *C.YRX_METADATA metadata []Metadata } @@ -108,29 +108,13 @@ func newRule(cRule *C.YRX_RULE) *Rule { identifier := C.GoStringN((*C.char)(unsafe.Pointer(str)), C.int(len)) - var metadata []Metadata - - var buf *C.YRX_BUFFER - if C.yrx_rule_metadata_as_json(cRule, &buf) == C.SUCCESS { - var tmp [][]interface{} - if err := json.Unmarshal(C.GoBytes(unsafe.Pointer(buf.data), C.int(buf.length)), &tmp); err != nil { - panic("yrx_rule_metadata_as_json") - } - for _, v := range tmp { - metadata = append(metadata, Metadata{ - Identifier: v[0].(string), - Value: v[1], - }) - } - defer C.yrx_buffer_destroy(buf) - } - rule := &Rule{ namespace, identifier, C.yrx_rule_patterns(cRule), nil, - metadata, + C.yrx_rule_metadata(cRule), + nil, } runtime.SetFinalizer(rule, (*Rule).destroy) @@ -139,6 +123,9 @@ func newRule(cRule *C.YRX_RULE) *Rule { func (r *Rule) destroy() { C.yrx_patterns_destroy(r.cPatterns) + if r.cMetadata != nil { + C.yrx_metadata_destroy(r.cMetadata) + } runtime.SetFinalizer(r, nil) } @@ -154,9 +141,43 @@ func (r *Rule) Namespace() string { // Metadata returns the rule's metadata func (r *Rule) Metadata() []Metadata { + // if this method was called before, return the metadata already cached. + if r.metadata != nil { + return r.metadata + } + + numMetadata := int(r.cMetadata.num_entries) + cMetadata := unsafe.Slice(r.cMetadata.entries, numMetadata) + r.metadata = make([]Metadata, numMetadata) + + for i, metadata := range cMetadata { + r.metadata[i].Identifier = C.GoString(metadata.identifier) + switch metadata.value_type { + case C.I64: + r.metadata[i].Value = int64(*(*C.int)(unsafe.Pointer(&metadata.value))) + case C.F64: + r.metadata[i].Value = float64(*(*C.double)(unsafe.Pointer(&metadata.value))) + case C.BOOLEAN: + r.metadata[i].Value = metadata.value[0] != 0 + case C.STRING: + r.metadata[i].Value = C.GoString(*(**C.char)(unsafe.Pointer(&metadata.value))) + case C.BYTES: + r.metadata[i].Value = C.GoBytes( + unsafe.Pointer(&metadata.value[8]), + C.int(*(*C.size_t)(unsafe.Pointer(&metadata.value))), + ) + } + } + return r.metadata } +// Metadata represents a metadata in a Rule. +type Metadata struct { + Identifier string + Value interface{} +} + // Patterns returns the patterns defined by this rule. func (r *Rule) Patterns() []Pattern { // If this method was called before, return the patterns already cached. @@ -189,12 +210,6 @@ func (r *Rule) Patterns() []Pattern { return r.patterns } -// Metadata represents a metadata in a Rule. -type Metadata struct { - Identifier string - Value interface{} -} - // Pattern represents a pattern in a Rule. type Pattern struct { identifier string diff --git a/go/scanner_test.go b/go/scanner_test.go index d92f90ff4..9bf302b5d 100644 --- a/go/scanner_test.go +++ b/go/scanner_test.go @@ -10,30 +10,24 @@ import ( ) func TestScanner1(t *testing.T) { - r, _ := Compile("rule t { meta: some_int = 1 some_bool = true condition: true }") + r, _ := Compile("rule t { condition: true }") s := NewScanner(r) matchingRules, _ := s.Scan([]byte{}) assert.Len(t, matchingRules, 1) assert.Equal(t, "t", matchingRules[0].Identifier()) assert.Equal(t, "default", matchingRules[0].Namespace()) - assert.Equal(t, "some_int", matchingRules[0].Metadata()[0].Identifier) - assert.Equal(t, float64(1), matchingRules[0].Metadata()[0].Value) - assert.Equal(t, "some_bool", matchingRules[0].Metadata()[1].Identifier) - assert.Equal(t, true, matchingRules[0].Metadata()[1].Value) assert.Len(t, matchingRules[0].Patterns(), 0) } func TestScanner2(t *testing.T) { - r, _ := Compile(`rule t { meta: some_string = "hello" strings: $bar = "bar" condition: $bar }`) + r, _ := Compile(`rule t { strings: $bar = "bar" condition: $bar }`) s := NewScanner(r) matchingRules, _ := s.Scan([]byte("foobar")) assert.Len(t, matchingRules, 1) assert.Equal(t, "t", matchingRules[0].Identifier()) assert.Equal(t, "default", matchingRules[0].Namespace()) - assert.Equal(t, "some_string", matchingRules[0].Metadata()[0].Identifier) - assert.Equal(t, "hello", matchingRules[0].Metadata()[0].Value) assert.Len(t, matchingRules[0].Patterns(), 1) assert.Equal(t, "$bar", matchingRules[0].Patterns()[0].Identifier()) @@ -87,3 +81,19 @@ func TestScannerTimeout(t *testing.T) { _, err := s.Scan(bytes.Repeat([]byte("a"), 10000)) assert.ErrorIs(t, err, ErrTimeout) } + +func TestScannerMetadata(t *testing.T) { + r, _ := Compile(`rule t { meta: some_int = 1 some_float = 2.3034 some_bool = true some_string = "hello" condition: true }`) + s := NewScanner(r) + matchingRules, _ := s.Scan([]byte{}) + + assert.Len(t, matchingRules, 1) + assert.Equal(t, "some_int", matchingRules[0].Metadata()[0].Identifier) + assert.Equal(t, int64(1), matchingRules[0].Metadata()[0].Value) + assert.Equal(t, "some_float", matchingRules[0].Metadata()[1].Identifier) + assert.Equal(t, float64(2.3034), matchingRules[0].Metadata()[1].Value) + assert.Equal(t, "some_bool", matchingRules[0].Metadata()[2].Identifier) + assert.Equal(t, true, matchingRules[0].Metadata()[2].Value) + assert.Equal(t, "some_string", matchingRules[0].Metadata()[3].Identifier) + assert.Equal(t, "hello", matchingRules[0].Metadata()[3].Value) +} From fa8d4e356c4b5f770f6178d9082d9c770adee7c8 Mon Sep 17 00:00:00 2001 From: "Victor M. Alvarez" Date: Fri, 24 May 2024 20:03:51 +0200 Subject: [PATCH 3/4] refactor: access metadata values in a safer manner. --- go/main.go | 38 ++++++++++++++++++++++++++++++++------ go/scanner_test.go | 13 ++++++++++++- 2 files changed, 44 insertions(+), 7 deletions(-) diff --git a/go/main.go b/go/main.go index 26de063d9..a8fe7a454 100644 --- a/go/main.go +++ b/go/main.go @@ -1,9 +1,30 @@ // Package yara_x provides Go bindings to the YARA-X library. package yara_x +import "C" // #cgo !static_link pkg-config: yara_x_capi // #cgo static_link pkg-config: --static yara_x_capi // #include +// +// uint64_t meta_i64(void* value) { +// return ((YRX_METADATA_VALUE*) value)->i64; +// } +// +// double meta_f64(void* value) { +// return ((YRX_METADATA_VALUE*) value)->f64; +// } +// +// bool meta_bool(void* value) { +// return ((YRX_METADATA_VALUE*) value)->boolean; +// } +// +// char* meta_str(void* value) { +// return ((YRX_METADATA_VALUE*) value)->string; +// } +// +// YRX_METADATA_BYTES* meta_bytes(void* value) { +// return &(((YRX_METADATA_VALUE*) value)->bytes); +// } import "C" import ( "errors" @@ -154,17 +175,22 @@ func (r *Rule) Metadata() []Metadata { r.metadata[i].Identifier = C.GoString(metadata.identifier) switch metadata.value_type { case C.I64: - r.metadata[i].Value = int64(*(*C.int)(unsafe.Pointer(&metadata.value))) + r.metadata[i].Value = int64( + C.meta_i64(unsafe.Pointer(&metadata.value))) case C.F64: - r.metadata[i].Value = float64(*(*C.double)(unsafe.Pointer(&metadata.value))) + r.metadata[i].Value = float64( + C.meta_f64(unsafe.Pointer(&metadata.value))) case C.BOOLEAN: - r.metadata[i].Value = metadata.value[0] != 0 + r.metadata[i].Value = bool( + C.meta_bool(unsafe.Pointer(&metadata.value))) case C.STRING: - r.metadata[i].Value = C.GoString(*(**C.char)(unsafe.Pointer(&metadata.value))) + r.metadata[i].Value = C.GoString( + C.meta_str(unsafe.Pointer(&metadata.value))) case C.BYTES: + bytes := C.meta_bytes(unsafe.Pointer(&metadata.value)) r.metadata[i].Value = C.GoBytes( - unsafe.Pointer(&metadata.value[8]), - C.int(*(*C.size_t)(unsafe.Pointer(&metadata.value))), + unsafe.Pointer(bytes.data), + C.int(bytes.length), ) } } diff --git a/go/scanner_test.go b/go/scanner_test.go index 9bf302b5d..c2d2b58f2 100644 --- a/go/scanner_test.go +++ b/go/scanner_test.go @@ -83,7 +83,16 @@ func TestScannerTimeout(t *testing.T) { } func TestScannerMetadata(t *testing.T) { - r, _ := Compile(`rule t { meta: some_int = 1 some_float = 2.3034 some_bool = true some_string = "hello" condition: true }`) + r, _ := Compile(`rule t { + meta: + some_int = 1 + some_float = 2.3034 + some_bool = true + some_string = "hello" + some_bytes = "\x00\x01\x02" + condition: + true + }`) s := NewScanner(r) matchingRules, _ := s.Scan([]byte{}) @@ -96,4 +105,6 @@ func TestScannerMetadata(t *testing.T) { assert.Equal(t, true, matchingRules[0].Metadata()[2].Value) assert.Equal(t, "some_string", matchingRules[0].Metadata()[3].Identifier) assert.Equal(t, "hello", matchingRules[0].Metadata()[3].Value) + assert.Equal(t, "some_bytes", matchingRules[0].Metadata()[4].Identifier) + assert.Equal(t, []byte{0, 1, 2}, matchingRules[0].Metadata()[4].Value) } From 6b8e82b26896c080ab6cdeb614f5f936be000099 Mon Sep 17 00:00:00 2001 From: "Victor M. Alvarez" Date: Fri, 24 May 2024 20:05:35 +0200 Subject: [PATCH 4/4] chore: remove unnecessary import statement --- go/main.go | 1 - 1 file changed, 1 deletion(-) diff --git a/go/main.go b/go/main.go index a8fe7a454..56f950b9b 100644 --- a/go/main.go +++ b/go/main.go @@ -1,6 +1,5 @@ // Package yara_x provides Go bindings to the YARA-X library. package yara_x -import "C" // #cgo !static_link pkg-config: yara_x_capi // #cgo static_link pkg-config: --static yara_x_capi