diff --git a/main.go b/main.go index c86cc26..b4b2fd1 100644 --- a/main.go +++ b/main.go @@ -9,7 +9,6 @@ import ( config "github.com/kyallanum/athena/v1.0.0/models/config" library "github.com/kyallanum/athena/v1.0.0/models/library" logs "github.com/kyallanum/athena/v1.0.0/models/logs" - "github.com/kyallanum/athena/v1.0.0/utils" ) func errCheck(err error) { @@ -46,31 +45,31 @@ func parseFlags(configFile *string, logFile *string) error { return nil } -func resolveLogFile(contents *logs.LogFile, config *config.Configuration) (*library.Library, error) { +func resolveLogFile(contents *logs.LogFile, configuration *config.Configuration) (*library.Library, error) { wrapError := func(err error) error { return fmt.Errorf("unable to resolve log file: \n\t%w", err) } if contents == nil || contents.Len() == 0 { return nil, fmt.Errorf("log file contains no contents") - } else if config == nil || (config.Name == "" && config.Rules == nil) { + } else if configuration == nil || (configuration.Name == "" && configuration.Rules == nil) { return nil, fmt.Errorf("configuration file has no contents") - } else if config.Name == "" { + } else if configuration.Name == "" { return nil, fmt.Errorf("configuration file contains no log name") - } else if config.Rules == nil || len(config.Rules) == 0 { + } else if configuration.Rules == nil || len(configuration.Rules) == 0 { return nil, fmt.Errorf("configuration does not have any rules") } - ret_library := library.Library.New(library.Library{}, config.Name) + ret_library := library.Library.New(library.Library{}, configuration.Name) fmt.Println("Resolving Log File") - for i := 0; i < len(config.Rules); i++ { - currentRuleData, err := utils.ResolveRule(contents, &config.Rules[i]) + for i := 0; i < len(configuration.Rules); i++ { + currentRuleData, err := config.ResolveRule(contents, &configuration.Rules[i]) if err != nil { return nil, wrapError(err) } - ret_library.AddRuleData(config.Rules[i].Name, currentRuleData) + ret_library.AddRuleData(configuration.Rules[i].Name, currentRuleData) } fmt.Println("Log File Resolved") diff --git a/models/config/configuration.go b/models/config/configuration.go index be60cec..a0163cc 100644 --- a/models/config/configuration.go +++ b/models/config/configuration.go @@ -3,7 +3,6 @@ package models import ( "encoding/json" "fmt" - "regexp" ) // A Configuration struct represents the top level of a JSON configuration file. @@ -39,7 +38,7 @@ type Rule struct { func (config *Configuration) TranslateConfiguration() error { for ruleIndex, currentRule := range config.Rules { for searchTermIndex, currentSearchTerm := range currentRule.SearchTerms { - err := translateRegex(¤tSearchTerm) + err := translateConfigurationNamedGroups(¤tSearchTerm) if err != nil { return err } @@ -49,25 +48,6 @@ func (config *Configuration) TranslateConfiguration() error { return nil } -func translateRegex(regex *string) error { - if *regex == "" { - return fmt.Errorf("empty search terms are not allowed") - } - - defer func() { - if err := recover(); err != nil { - panic(fmt.Errorf("unable to translate regex: \"%s\" for Go standards, this is most likely an internal error: \n\t%s", *regex, err.(string))) - } - }() - - regexAddGolangGroupName := `(\(\?)(\<[\w\W]+?\>)` - compiledRegex := regexp.MustCompile(regexAddGolangGroupName) - - *regex = compiledRegex.ReplaceAllString(*regex, "${1}P${2}") - - return nil -} - func CreateConfiguration(source string) (config *Configuration, err error) { wrapError := func(err error) error { return fmt.Errorf("unable to create configuration object: \n\t%w", err) diff --git a/models/config/configuration_test.go b/models/config/configuration_test.go index 310622b..d3d3fe3 100644 --- a/models/config/configuration_test.go +++ b/models/config/configuration_test.go @@ -9,11 +9,13 @@ import ( "reflect" "strings" "testing" + + logs "github.com/kyallanum/athena/v1.0.0/models/logs" ) func TestTranslateRegex(t *testing.T) { regexToTranslate := "(?)" - err := translateRegex(®exToTranslate) + err := translateConfigurationNamedGroups(®exToTranslate) if err != nil { t.Errorf("An error occurred while attempting to translate regex: \n\t%v", err) @@ -26,7 +28,7 @@ func TestTranslateRegex(t *testing.T) { func TestTranslateRegexEmptyString(t *testing.T) { regexToTranslate := "" - err := translateRegex(®exToTranslate) + err := translateConfigurationNamedGroups(®exToTranslate) if err.Error() != "empty search terms are not allowed" { t.Errorf("Error was not properly returned when checking for empty string.") @@ -135,3 +137,41 @@ func TestCreateConfigurationFromWeb(t *testing.T) { }) } } + +func TestResolveRule(t *testing.T) { + os.Stdout, _ = os.Open(os.DevNull) + defer os.Stdout.Close() + + logFile, _ := logs.LoadLogFile("../../examples/apt-term.log") + + currentConfig, _ := CreateConfiguration("../../examples/apt-term-config.json") + + currentRule := currentConfig.Rules[0] + + ruleData, err := ResolveRule(logFile, ¤tRule) + if err != nil { + t.Errorf("An error was returned when one should not have been: \n\t%s", err.Error()) + } + + if reflect.TypeOf(ruleData).String() != "*models.RuleData" { + t.Errorf("The incorrect datatype was not returned: \n\t%s", reflect.TypeOf(ruleData).String()) + } +} + +func TestResolveRuleBadRule(t *testing.T) { + defer func() { + if err := recover(); err == nil { + t.Errorf("An error was not returned when one should have been.") + } else if (err.(error)).Error() != "unable to resolve search terms for rule : \n\truntime error: index out of range [0] with length 0" { + t.Errorf("%s", err.(error).Error()) + } + }() + os.Stdout, _ = os.Open(os.DevNull) + defer os.Stdout.Close() + + logFile, _ := logs.LoadLogFile("../examples/apt-term.log") + + currentRule := &Rule{} + + ResolveRule(logFile, currentRule) +} diff --git a/models/config/regex_utils.go b/models/config/regex_utils.go new file mode 100644 index 0000000..a2c6d07 --- /dev/null +++ b/models/config/regex_utils.go @@ -0,0 +1,102 @@ +package models + +import ( + "fmt" + "regexp" + "strings" + + library "github.com/kyallanum/athena/v1.0.0/models/library" +) + +// func resolveLine determines whether the current log line matches a +// matches a given log line +func resolveLine(line string, regex string) *map[string]string { + defer func() { + if err := recover(); err != nil { + panic(fmt.Errorf("the provided regular expression cannot be compiled: \n\t%s", err.(string))) + } + }() + + currentRegexp := regexp.MustCompile(regex) + match := currentRegexp.FindStringSubmatch(line) + result := make(map[string]string) + + if len(match) > 0 { + for index, name := range currentRegexp.SubexpNames() { + if index != 0 && name != "" { + result[name] = match[index] + } + } + return &result + } + return nil +} + +func translateSearchTermReference(regex string, currentSearchTermData *library.SearchTermData) (string, error) { + defer func() { + if err := recover(); err != nil { + panic(fmt.Errorf("the search term could not be translated. this is most likely an internal error: \n\t%s", err.(string))) + } + }() + + // Matches the pattern: {{name_to_replace}} + nameExtractRegex := `(\{\{(?P[\w]+?)\}\})` + re := regexp.MustCompile(nameExtractRegex) + matches := re.FindAllStringSubmatch(regex, -1) + numMatches := len(matches) + + keysToLookup := []string{} + + // The first three groups are useless for this case. So get the last one. + for _, match := range matches { + keysToLookup = append(keysToLookup, match[2]) + } + + // Validate all strings to replace and then replace them one by one. + for i := 0; i < numMatches; i++ { + stringToReplace, err := currentSearchTermData.Value(strings.TrimSpace(strings.ToLower(keysToLookup[i]))) + if err != nil { + return "", fmt.Errorf("an error occurred when translating a search term reference. \n\tthe following key was not registered in a previous search term: %s", keysToLookup[i]) + } + stringToReplace = escapeSpecialCharacters(stringToReplace) + foundString := re.FindString(regex) + if foundString != "" { + regex = strings.Replace(regex, foundString, stringToReplace, 1) + } + } + + return regex, nil +} + +func escapeSpecialCharacters(regex string) string { + defer func() { + if err := recover(); err != nil { + panic(fmt.Errorf("a string could not be escaped. this is most likely an internal error: \n\t%s", err.(string))) + } + }() + charactersToEscape := `([\*\+\?\\\.\^\[\]\$\&\|]{1})` + re := regexp.MustCompile(charactersToEscape) + + regex = re.ReplaceAllString(regex, `\${1}`) + return regex +} + +func translateConfigurationNamedGroups(regex *string) error { + if *regex == "" { + return fmt.Errorf("empty search terms are not allowed") + } + + defer func() { + if err := recover(); err != nil { + panic(fmt.Errorf("unable to translate regex: \"%s\" for Go standards, this is most likely an internal error: \n\t%s", *regex, err.(string))) + } + }() + + // Matches the pattern (?) + regexAddGolangGroupName := `(\(\?)(\<[\w\W]+?\>)` + compiledRegex := regexp.MustCompile(regexAddGolangGroupName) + + *regex = compiledRegex.ReplaceAllString(*regex, "${1}P${2}") + + return nil +} diff --git a/models/config/regex_utils_test.go b/models/config/regex_utils_test.go new file mode 100644 index 0000000..857bd95 --- /dev/null +++ b/models/config/regex_utils_test.go @@ -0,0 +1,102 @@ +package models + +import ( + "reflect" + "testing" + + library "github.com/kyallanum/athena/v1.0.0/models/library" +) + +func TestResolveLine(t *testing.T) { + defer func() { + if err := recover(); err != nil { + t.Errorf("An error was returned improperly when calling resolveLine: \n\t%s", (err.(error)).Error()) + } + }() + line := "test line 1" + regex := "stuff" + + result := resolveLine(line, regex) + if result != nil { + t.Errorf("An incorrect object was returned when calling resolveLine") + } + + regex = `(?Pline \d+)` + result = resolveLine(line, regex) + + if reflect.TypeOf(result).String() != "*map[string]string" { + t.Errorf("resolveLine did not return the proper data type: \n\t%s", reflect.TypeOf(result).String()) + } +} + +func TestResolveLineBadRegex(t *testing.T) { + defer func() { + if err := recover(); err == nil { + t.Errorf("An error was not returned properly when it should have") + } else if (err.(error)).Error() != "the provided regular expression cannot be compiled: \n\tregexp: Compile(`(stuff`): error parsing regexp: missing closing ): `(stuff`" { + t.Errorf("The improper error was returned when calling with a bad regex: \n\t%s", (err.(error)).Error()) + } + }() + + line := "test line 1" + regex := "(stuff" + + _ = resolveLine(line, regex) +} + +func TestTranslateSearchTermReference(t *testing.T) { + defer func() { + if err := recover(); err != nil { + t.Errorf("An error was returned when it shouldn't have: \n\t%s", (err.(error).Error())) + } + }() + + regex := `Testing {{test}}` + st_data := library.SearchTermData.New(library.SearchTermData{}) + st_data.AddValue("test", "Test1") + + newRegex, err := translateSearchTermReference(regex, st_data) + if err != nil { + t.Errorf("An error was returned when it shouldn't have: \n\t%s", err.Error()) + } + + if newRegex != "Testing Test1" { + t.Errorf("TranslateSearchTermReference returned the wrong value: \n\t%s", newRegex) + } +} + +func TestTranslateSearchTermReferenceBadReference(t *testing.T) { + defer func() { + if err := recover(); err != nil { + t.Errorf("An error was returned when it shouldn't have: \n\t%s", (err.(error).Error())) + } + }() + + regex := `Testing {{test}}` + st_data := library.SearchTermData.New(library.SearchTermData{}) + st_data.AddValue("bad_test", "testing") + + _, err := translateSearchTermReference(regex, st_data) + if err == nil { + t.Errorf("An error was not returned when it should have.") + } + + if err.Error() != "an error occurred when translating a search term reference. \n\tthe following key was not registered in a previous search term: test" { + t.Errorf("An improper error was returned when attempting to translate search term reference: \n\t%s", err.Error()) + } +} + +func TestValidateString(t *testing.T) { + defer func() { + if err := recover(); err != nil { + t.Errorf("An error occurred when it shouldn't have: \n\t%s", (err.(error).Error())) + } + }() + + stringToValidate := `*+?\\.^[]$&|` + validatedString := escapeSpecialCharacters(stringToValidate) + + if validatedString != "\\*\\+\\?\\\\\\\\\\.\\^\\[\\]\\$\\&\\|" { + t.Errorf("String was not validated properly: %s", validatedString) + } +} diff --git a/models/config/rule_utils.go b/models/config/rule_utils.go new file mode 100644 index 0000000..950cdb5 --- /dev/null +++ b/models/config/rule_utils.go @@ -0,0 +1,43 @@ +package models + +import ( + "fmt" + + library "github.com/kyallanum/athena/v1.0.0/models/library" + logs "github.com/kyallanum/athena/v1.0.0/models/logs" +) + +func ResolveRule(contents *logs.LogFile, rule *Rule) (*library.RuleData, error) { + wrapError := func(err error) error { + return fmt.Errorf("unable to resolve rule %s: \n\t%w", rule.Name, err) + } + + allEntriesFound := false + linesResolved := []int{} + + currentRuleData := library.RuleData.New(library.RuleData{}) + + for !allEntriesFound { + currentSearchTermData, err := resolveSearchTerms(contents, rule, &linesResolved) + if err != nil { + return nil, wrapError(err) + } + + if len(currentSearchTermData.Keys()) != 0 { + currentRuleData.AppendSearchTermData(currentSearchTermData) + } else { + allEntriesFound = true + } + } + + for _, summaryLine := range rule.Summary { + summaryData, err := resolveSummaryLine(summaryLine, ¤tRuleData) + if err != nil { + return nil, wrapError(err) + } + for _, line := range summaryData { + currentRuleData.AppendSummaryData(line) + } + } + return ¤tRuleData, nil +} diff --git a/models/config/searchterm_utils.go b/models/config/searchterm_utils.go new file mode 100644 index 0000000..6d90dae --- /dev/null +++ b/models/config/searchterm_utils.go @@ -0,0 +1,71 @@ +package models + +import ( + "fmt" + "slices" + + library "github.com/kyallanum/athena/v1.0.0/models/library" + logs "github.com/kyallanum/athena/v1.0.0/models/logs" +) + +func resolveSearchTerms(logFile *logs.LogFile, rule *Rule, linesResolved *[]int) (*library.SearchTermData, error) { + wrapError := func(err error) error { + return fmt.Errorf("unable to resolve search terms for rule %s: \n\t%w", rule.Name, err) + } + + defer func() { + if err := recover(); err != nil { + panic(wrapError(fmt.Errorf("%s", err))) + } + }() + + currentSearchTermData := library.SearchTermData.New(library.SearchTermData{}) + currentSearchTerm := rule.SearchTerms[0] + searchTermTranslated := false + for fileIndex, searchTermIndex := 0, 0; fileIndex < logFile.Len() && searchTermIndex < len(rule.SearchTerms); fileIndex++ { + if slices.Contains(*linesResolved, fileIndex) { + continue + } + + if !searchTermTranslated { + newSearchTerm, err := translateSearchTermReference(currentSearchTerm, currentSearchTermData) + if err != nil { + return nil, wrapError(err) + } + currentSearchTerm = newSearchTerm + searchTermTranslated = true + } + + currentLine, err := logFile.LineAtIndex(fileIndex) + if err != nil { + return nil, wrapError(err) + } + + result := resolveLine(currentLine, currentSearchTerm) + if result == nil { + continue + } + + if rule.PrintLog { + fmt.Printf("%d: %s\n", fileIndex+1, currentLine) + } + + *linesResolved = append(*linesResolved, fileIndex) + + for key, value := range *result { + err := currentSearchTermData.AddValue(key, value) + if err != nil { + return nil, wrapError(err) + } + } + + searchTermIndex++ + if searchTermIndex == len(rule.SearchTerms) { + break + } + currentSearchTerm = rule.SearchTerms[searchTermIndex] + searchTermTranslated = false + } + + return currentSearchTermData, nil +} diff --git a/models/config/summary_utils.go b/models/config/summary_utils.go new file mode 100644 index 0000000..830414d --- /dev/null +++ b/models/config/summary_utils.go @@ -0,0 +1,82 @@ +package models + +import ( + "fmt" + "regexp" + "slices" + "strings" + + library "github.com/kyallanum/athena/v1.0.0/models/library" +) + +func summaryKeys(summaryLine string) [][]string { + defer func() { + if err := recover(); err != nil { + panic(fmt.Errorf("could not get summary keys. this is most likely an internal error: \n\t%s", err.(string))) + } + }() + + ret_keys := make([][]string, 0) + keyRegex := `\{\{(?P[\w]+)(\()(?P[\w]+)(\))\}\}` + re := regexp.MustCompile(keyRegex) + matches := re.FindAllStringSubmatch(summaryLine, -1) + + for _, match := range matches { + original := match[0] + operation := match[1] + key := match[3] + keyToAdd := []string{original, operation, key} + ret_keys = append(ret_keys, keyToAdd) + } + + return ret_keys +} + +func resolveSummaryLine(summaryLine string, ruleData *library.RuleData) ([]string, error) { + wrapError := func(err error) error { + return fmt.Errorf("unable to resolve summary line: \n\t%w", err) + } + + keys := summaryKeys(summaryLine) + if !isOneUniqueOperation(keys) { + return nil, fmt.Errorf("could not resolve summary. mixing operations is currently not implemented") + } + + ret_summary_line := make([]string, 0) + expanded := false + + for _, key := range keys { + operation, err := library.Operation(key[1], key[2]) + if err != nil { + return nil, wrapError(err) + } + + calculated, err := operation.CalculateOperation(*ruleData) + if err != nil { + return nil, wrapError(err) + } + + for i := 0; i < len(calculated); i++ { + // expand the first time + if !expanded { + ret_summary_line = append(ret_summary_line, summaryLine) + } + ret_summary_line[i] = strings.Replace(ret_summary_line[i], key[0], calculated[i], 1) + } + expanded = true + } + return ret_summary_line, nil +} + +func isOneUniqueOperation(keys [][]string) bool { + currentKeys := make([]string, 0) + for _, key := range keys { + currentKeys = append(currentKeys, key[1]) + } + + ret_unique := slices.CompactFunc(currentKeys, func(i, j string) bool { + return strings.EqualFold(i, j) + }) + + return len(ret_unique) == 1 +} diff --git a/models/library/operation_factory.go b/models/library/operation_factory.go index 82fb033..21e2e0e 100644 --- a/models/library/operation_factory.go +++ b/models/library/operation_factory.go @@ -1,5 +1,10 @@ package models +import ( + "fmt" + "strings" +) + type ISummaryOperation interface { CalculateOperation(ruleData RuleData) ([]string, error) Operation() string @@ -28,3 +33,14 @@ func (summaryKey *SummaryOperation) Key() string { func (summaryKey *SummaryOperation) SetKey(key string) { summaryKey.key = key } + +func Operation(operation string, key string) (ISummaryOperation, error) { + switch strings.ToLower(strings.TrimSpace(operation)) { + case "count": + return Count.New(Count{}, key), nil + case "print": + return Print.New(Print{}, key), nil + } + + return nil, fmt.Errorf("the given operation is not implemented: %s\n\t", operation) +}