Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[8.x](backport #42297) [winlogbeat] Several improvements to winlogbeat raw api #42298

Merged
merged 1 commit into from
Jan 13, 2025
Merged

[8.x](backport #42297) [winlogbeat] Several improvements to winlogbeat raw api #42298

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
63 changes: 43 additions & 20 deletions winlogbeat/sys/wineventlog/metadata_store.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@
package wineventlog

import (
"encoding/xml"
"fmt"
"strconv"
"strings"
Expand Down Expand Up @@ -51,10 +52,10 @@ type PublisherMetadataStore struct {
winevent.WinMeta

// Event ID to event metadata (message and event data param names).
Events map[uint16]*EventMetadata
Events map[uint32]*EventMetadata
// Event ID to map of fingerprints to event metadata. The fingerprint value
// is hash of the event data parameters count and types.
EventFingerprints map[uint16]map[uint64]*EventMetadata
EventFingerprints map[uint32]map[uint64]*EventMetadata
// Stores used messages by their ID. Message can be found in events as references
// such as %%1111. This need to be formatted the first time, and they are stored
// from that point after.
Expand All @@ -71,7 +72,7 @@ func NewPublisherMetadataStore(session EvtHandle, provider string, locale uint32
}
store := &PublisherMetadataStore{
Metadata: md,
EventFingerprints: map[uint16]map[uint64]*EventMetadata{},
EventFingerprints: map[uint32]map[uint64]*EventMetadata{},
MessagesByID: map[uint32]string{},
log: log.With("publisher", provider),
}
Expand Down Expand Up @@ -102,8 +103,8 @@ func NewEmptyPublisherMetadataStore(provider string, log *logp.Logger) *Publishe
Levels: map[uint8]string{},
Tasks: map[uint16]string{},
},
Events: map[uint16]*EventMetadata{},
EventFingerprints: map[uint16]map[uint64]*EventMetadata{},
Events: map[uint32]*EventMetadata{},
EventFingerprints: map[uint32]map[uint64]*EventMetadata{},
MessagesByID: map[uint32]string{},
log: log.With("publisher", provider, "empty", true),
}
Expand Down Expand Up @@ -137,7 +138,7 @@ func (s *PublisherMetadataStore) initOpcodes() error {
if val == "" {
val = opcodeMeta.Name
}
s.Opcodes[uint8(opcodeMeta.Mask)] = val
s.Opcodes[uint8(opcodeMeta.Opcode)] = val
}
return nil
}
Expand Down Expand Up @@ -182,23 +183,24 @@ func (s *PublisherMetadataStore) initEvents() error {
}
defer itr.Close()

s.Events = map[uint16]*EventMetadata{}
s.Events = map[uint32]*EventMetadata{}
for itr.Next() {
evt, err := newEventMetadataFromPublisherMetadata(itr, s.Metadata)
if err != nil {
s.log.Warnw("Failed to read event metadata from publisher. Continuing to next event.",
"error", err)
continue
}
s.Events[evt.EventID] = evt
s.Events[getEventCombinedID(evt.EventID, evt.Version)] = evt
}
return itr.Err()
}

func (s *PublisherMetadataStore) getEventMetadata(eventID uint16, eventDataFingerprint uint64, eventHandle EvtHandle) *EventMetadata {
func (s *PublisherMetadataStore) getEventMetadata(eventID uint16, version uint8, eventDataFingerprint uint64, eventHandle EvtHandle) *EventMetadata {
// Use a read lock to get a cached value.
s.mutex.RLock()
fingerprints, found := s.EventFingerprints[eventID]
combinedID := getEventCombinedID(eventID, version)
fingerprints, found := s.EventFingerprints[combinedID]
if found {
em, found := fingerprints[eventDataFingerprint]
if found {
Expand All @@ -212,10 +214,10 @@ func (s *PublisherMetadataStore) getEventMetadata(eventID uint16, eventDataFinge
s.mutex.Lock()
defer s.mutex.Unlock()

fingerprints, found = s.EventFingerprints[eventID]
fingerprints, found = s.EventFingerprints[combinedID]
if !found {
fingerprints = map[uint64]*EventMetadata{}
s.EventFingerprints[eventID] = fingerprints
s.EventFingerprints[combinedID] = fingerprints
}

em, found := fingerprints[eventDataFingerprint]
Expand All @@ -233,7 +235,7 @@ func (s *PublisherMetadataStore) getEventMetadata(eventID uint16, eventDataFinge
// metadata then we just associate the fingerprint with a pointer to the
// providers metadata for the event ID.

defaultEM := s.Events[eventID]
defaultEM := s.Events[combinedID]

// Use XML to get the parameters names.
em, err := newEventMetadataFromEventHandle(s.Metadata, eventHandle)
Expand All @@ -250,6 +252,15 @@ func (s *PublisherMetadataStore) getEventMetadata(eventID uint16, eventDataFinge
return defaultEM
}

// The first time we need to identify if the event has event data or
// user data from a handle, since this information is not available
// from the metadata or anywhere else. It is not ideal to update the defaultEM
// here but there is no way around it at the moment.
if em.EventData.IsUserData {
defaultEM.EventData.IsUserData = true
defaultEM.EventData.Name = em.EventData.Name
}

// Are the parameters the same as what the provider metadata listed?
// (This ignores the message values.)
if em.equal(defaultEM) {
Expand Down Expand Up @@ -324,13 +335,18 @@ func (s *PublisherMetadataStore) Close() error {
return nil
}

type EventDataParams struct {
IsUserData bool
Name xml.Name
Params []EventData
}

type EventMetadata struct {
EventID uint16 // Event ID.
Version uint8 // Event format version.
MsgStatic string // Used when the message has no parameters.
MsgTemplate *template.Template `json:"-"` // Template that expects an array of values as its data.
EventData []EventData // Names of parameters from XML template.
HasUserData bool // Event has a UserData section or not.
EventData EventDataParams // Names of parameters from XML template.
}

// newEventMetadataFromEventHandle collects metadata about an event type using
Expand All @@ -354,12 +370,13 @@ func newEventMetadataFromEventHandle(publisher *PublisherMetadata, eventHandle E
}
if len(event.EventData.Pairs) > 0 {
for _, pair := range event.EventData.Pairs {
em.EventData = append(em.EventData, EventData{Name: pair.Key})
em.EventData.Params = append(em.EventData.Params, EventData{Name: pair.Key})
}
} else {
em.HasUserData = true
em.EventData.IsUserData = true
em.EventData.Name = event.UserData.Name
for _, pair := range event.UserData.Pairs {
em.EventData = append(em.EventData, EventData{Name: pair.Key})
em.EventData.Params = append(em.EventData.Params, EventData{Name: pair.Key})
}
}

Expand Down Expand Up @@ -435,7 +452,7 @@ func (em *EventMetadata) initEventDataTemplate(itr *EventMetadataIterator) error
kv.Name = eventDataNameTransform.Replace(kv.Name)
}

em.EventData = tmpl.Data
em.EventData.Params = tmpl.Data
return nil
}

Expand Down Expand Up @@ -477,6 +494,10 @@ func (em *EventMetadata) setMessage(msg string) error {
return nil
}

func getEventCombinedID(eventID uint16, version uint8) uint32 {
return (uint32(eventID) << 16) | uint32(version)
}

// containsTemplatedValues traverses the template nodes to check if there are
// any dynamic values.
func containsTemplatedValues(tmpl *template.Template) bool {
Expand Down Expand Up @@ -513,7 +534,9 @@ func (em *EventMetadata) equal(other *EventMetadata) bool {

return em.EventID == other.EventID &&
em.Version == other.Version &&
eventDataNamesEqual(em.EventData, other.EventData)
em.EventData.IsUserData == other.EventData.IsUserData &&
em.EventData.Name == other.EventData.Name &&
eventDataNamesEqual(em.EventData.Params, other.EventData.Params)
}

type publisherMetadataCache struct {
Expand Down
9 changes: 7 additions & 2 deletions winlogbeat/sys/wineventlog/publisher_metadata.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -198,7 +198,7 @@
if err != nil {
return nil, err
}
messageID := v.(uint32)

Check failure on line 201 in winlogbeat/sys/wineventlog/publisher_metadata.go

View workflow job for this annotation

GitHub Actions / lint (windows) 

Error return value is not checked (errcheck)

// The value is -1 if the keyword did not specify a message attribute.
var message string
Expand All @@ -213,13 +213,13 @@
if err != nil {
return nil, err
}
name := v.(string)

Check failure on line 216 in winlogbeat/sys/wineventlog/publisher_metadata.go

View workflow job for this annotation

GitHub Actions / lint (windows) 

Error return value is not checked (errcheck)

v, err = EvtGetObjectArrayProperty(arrayHandle, EvtPublisherMetadataKeywordValue, index)
if err != nil {
return nil, err
}
valueMask := v.(uint64)

Check failure on line 222 in winlogbeat/sys/wineventlog/publisher_metadata.go

View workflow job for this annotation

GitHub Actions / lint (windows) 

Error return value is not checked (errcheck)

return &MetadataKeyword{
Name: name,
Expand All @@ -231,7 +231,8 @@

type MetadataOpcode struct {
Name string
Mask uint32
Opcode uint16
Task uint16
MessageID uint32
Message string
}
Expand Down Expand Up @@ -292,11 +293,15 @@
if err != nil {
return nil, err
}
// Mask high word contains the opcode value and the low word contains the task to which it belongs.
// If the low word is zero, the opcode is defined globally; otherwise, the opcode is task specific.
// Use the low word value to determine the task that defines the opcode.
valueMask := v.(uint32)

return &MetadataOpcode{
Name: name,
Mask: valueMask,
Opcode: uint16((valueMask >> 16) & 0xFFFF),
Task: uint16(valueMask & 0xFFFF),
MessageID: messageID,
Message: message,
}, nil
Expand Down Expand Up @@ -548,7 +553,7 @@
func NewEventMetadataIterator(publisher *PublisherMetadata) (*EventMetadataIterator, error) {
eventMetadataEnumHandle, err := _EvtOpenEventMetadataEnum(publisher.Handle, 0)
if err != nil && err != windows.ERROR_FILE_NOT_FOUND { //nolint:errorlint // Bad linter! This is always errno or nil.
return nil, fmt.Errorf("failed to open event metadata enumerator with EvtOpenEventMetadataEnum: %w (%#v)", err, err)

Check failure on line 556 in winlogbeat/sys/wineventlog/publisher_metadata.go

View workflow job for this annotation

GitHub Actions / lint (windows) 

non-wrapping format verb for fmt.Errorf. Use `%w` to format errors (errorlint)
}

return &EventMetadataIterator{
Expand Down
13 changes: 7 additions & 6 deletions winlogbeat/sys/wineventlog/renderer.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -126,7 +126,7 @@ func (r *Renderer) Render(handle EvtHandle) (*winevent.Event, string, error) {
}

// Load cached event metadata or try to bootstrap it from the event's XML.
eventMeta := md.getEventMetadata(uint16(event.EventIdentifier.ID), fingerprint, handle)
eventMeta := md.getEventMetadata(uint16(event.EventIdentifier.ID), uint8(event.Version), fingerprint, handle)

// Associate key names with the event data values.
r.addEventData(eventMeta, eventData, event)
Expand Down Expand Up @@ -304,13 +304,13 @@ func (r *Renderer) addEventData(evtMeta *EventMetadata, values []interface{}, ev
r.log.Warnw("Event metadata not found.",
"provider", event.Provider.Name,
"event_id", event.EventIdentifier.ID)
} else if len(values) != len(evtMeta.EventData) {
} else if len(values) != len(evtMeta.EventData.Params) {
r.log.Warnw("The number of event data parameters doesn't match the number "+
"of parameters in the template.",
"provider", event.Provider.Name,
"event_id", event.EventIdentifier.ID,
"event_parameter_count", len(values),
"template_parameter_count", len(evtMeta.EventData),
"template_parameter_count", len(evtMeta.EventData.Params),
"template_version", evtMeta.Version,
"event_version", event.Version)
}
Expand All @@ -322,8 +322,8 @@ func (r *Renderer) addEventData(evtMeta *EventMetadata, values []interface{}, ev
// updated). If software was updated it could also be that this cached
// template is now stale.
paramName := func(idx int) string {
if evtMeta != nil && idx < len(evtMeta.EventData) {
return evtMeta.EventData[idx].Name
if evtMeta != nil && idx < len(evtMeta.EventData.Params) {
return evtMeta.EventData.Params[idx].Name
}
return "param" + strconv.Itoa(idx)
}
Expand All @@ -346,7 +346,8 @@ func (r *Renderer) addEventData(evtMeta *EventMetadata, values []interface{}, ev
}
}

if evtMeta != nil && evtMeta.HasUserData {
if evtMeta != nil && evtMeta.EventData.IsUserData {
event.UserData.Name = evtMeta.EventData.Name
event.UserData.Pairs = pairs
} else {
event.EventData.Pairs = pairs
Expand Down
Loading