From ef3d66a97c02b7db100104ed3434643996f4462b Mon Sep 17 00:00:00 2001 From: Aditya Date: Wed, 11 Jan 2023 16:08:27 +0545 Subject: [PATCH 1/8] File backend - draft I --- api/logs/logs.go | 6 ++++ pkg/config.go | 18 ++++++++--- pkg/files/search.go | 73 +++++++++++++++++++++++++++++++++++++++++++++ samples/config.yaml | 9 +++++- 4 files changed, 101 insertions(+), 5 deletions(-) create mode 100644 pkg/files/search.go diff --git a/api/logs/logs.go b/api/logs/logs.go index 69ea31d..7ff5a52 100644 --- a/api/logs/logs.go +++ b/api/logs/logs.go @@ -19,6 +19,7 @@ type SearchBackend struct { Routes []SearchRoute `json:"routes,omitempty"` Backend SearchAPI `json:"-"` Kubernetes *KubernetesSearchBackend `json:"kubernetes,omitempty"` + Files []FileSearchBackend `json:"file,omitempty" yaml:"file,omitempty"` } type SearchRoute struct { @@ -34,6 +35,11 @@ type KubernetesSearchBackend struct { Namespace string `json:"namespace,omitempty"` } +type FileSearchBackend struct { + Labels map[string]string `yaml:"labels,omitempty"` + Paths []string `yaml:"path,omitempty"` +} + type SearchParams struct { // Limit is the maximum number of results to return. Limit int64 `json:"limit,omitempty"` diff --git a/pkg/config.go b/pkg/config.go index ed09875..46d0c4d 100644 --- a/pkg/config.go +++ b/pkg/config.go @@ -1,26 +1,28 @@ package pkg import ( - "io/ioutil" + "os" "github.com/flanksource/flanksource-ui/apm-hub/api/logs" + "github.com/flanksource/flanksource-ui/apm-hub/pkg/files" k8s "github.com/flanksource/flanksource-ui/apm-hub/pkg/kubernetes" "github.com/flanksource/kommons" "gopkg.in/yaml.v3" ) -//Initilize all the backends mentioned in the config +// Initilize all the backends mentioned in the config // GlobalBackends, error func ParseConfig(kommonsClient *kommons.Client, configFile string) ([]logs.SearchBackend, error) { searchConfig := &logs.SearchConfig{} backends := []logs.SearchBackend{} - data, err := ioutil.ReadFile(configFile) + data, err := os.ReadFile(configFile) if err != nil { return nil, err } if err := yaml.Unmarshal(data, searchConfig); err != nil { return nil, err } + for _, backend := range searchConfig.Backends { if backend.Kubernetes != nil { client, err := k8s.GetKubeClient(kommonsClient, backend.Kubernetes) @@ -30,8 +32,16 @@ func ParseConfig(kommonsClient *kommons.Client, configFile string) ([]logs.Searc backend.Backend = &k8s.KubernetesSearch{ Client: client, } + backends = append(backends, backend) + } + + if len(backend.Files) != 0 { + backend.Backend = &files.FileSearch{ + FilesBackend: backend.Files, + } + backends = append(backends, backend) } - backends = append(backends, backend) } + return backends, nil } diff --git a/pkg/files/search.go b/pkg/files/search.go new file mode 100644 index 0000000..90fb1d7 --- /dev/null +++ b/pkg/files/search.go @@ -0,0 +1,73 @@ +package files + +import ( + "bufio" + "fmt" + "os" + "strings" + + "github.com/flanksource/flanksource-ui/apm-hub/api/logs" +) + +type FileSearch struct { + FilesBackend []logs.FileSearchBackend +} + +func (t *FileSearch) Search(q *logs.SearchParams) (r logs.SearchResults, err error) { + var res logs.SearchResults + + for _, b := range t.FilesBackend { + if !matchQueryLabels(q.Labels, b.Labels) { + continue + } + + files, err := readFileLines(b.Paths) + if err != nil { + return res, fmt.Errorf("readFileLines(); %w", err) + } + + for _, content := range files { + res.Results = append(res.Results, content...) + } + } + + // TODO: Need to implement pagination but is it per file? + + return res, nil +} + +type logsPerFile map[string][]logs.Result + +// readFileLines will take a list of file paths +// and then return each lines of those files. +func readFileLines(paths []string) (logsPerFile, error) { + fileContents := make(logsPerFile, len(paths)) + for _, path := range paths { + content, err := os.Open(path) + if err != nil { + return nil, fmt.Errorf("error reading file_path=%s; %w", path, err) + } + + scanner := bufio.NewScanner(content) + for scanner.Scan() { + fileContents[path] = append(fileContents[path], logs.Result{ + // Id: , guess I can ignore the Id at this stage + // Time: , not sure how to reliably get the time here. This varies based on the log type. + // Labels: , all the records will have the same labels. Is it necessary to add it here? + Message: strings.TrimSpace(scanner.Text()), + }) + } + } + + return fileContents, nil +} + +func matchQueryLabels(want, have map[string]string) bool { + for label, val := range want { + if val != have[label] { + return false + } + } + + return true +} diff --git a/samples/config.yaml b/samples/config.yaml index 37dd7c0..25f44a1 100644 --- a/samples/config.yaml +++ b/samples/config.yaml @@ -1,3 +1,10 @@ backends: - kubernetes: - kubeconfig: \ No newline at end of file + kubeconfig: + + - file: + - labels: + name: acmehost + type: Nginx + path: + - /var/local/cyza/nginx/log/access.log From 70b5976cadec7c7a45e83b232082ba331d214f45 Mon Sep 17 00:00:00 2001 From: Aditya Date: Tue, 17 Jan 2023 12:32:22 +0545 Subject: [PATCH 2/8] Use last modified date of the file as the timestamp of the log. --- pkg/files/search.go | 25 ++++++++++++++----------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/pkg/files/search.go b/pkg/files/search.go index 90fb1d7..bf30185 100644 --- a/pkg/files/search.go +++ b/pkg/files/search.go @@ -5,6 +5,7 @@ import ( "fmt" "os" "strings" + "time" "github.com/flanksource/flanksource-ui/apm-hub/api/logs" ) @@ -21,9 +22,9 @@ func (t *FileSearch) Search(q *logs.SearchParams) (r logs.SearchResults, err err continue } - files, err := readFileLines(b.Paths) + files, err := readFilesLines(b.Paths) if err != nil { - return res, fmt.Errorf("readFileLines(); %w", err) + return res, fmt.Errorf("readFilesLines(); %w", err) } for _, content := range files { @@ -31,28 +32,30 @@ func (t *FileSearch) Search(q *logs.SearchParams) (r logs.SearchResults, err err } } - // TODO: Need to implement pagination but is it per file? - return res, nil } type logsPerFile map[string][]logs.Result -// readFileLines will take a list of file paths +// readFilesLines will take a list of file paths // and then return each lines of those files. -func readFileLines(paths []string) (logsPerFile, error) { +func readFilesLines(paths []string) (logsPerFile, error) { fileContents := make(logsPerFile, len(paths)) for _, path := range paths { - content, err := os.Open(path) + fInfo, err := os.Stat(path) + if err != nil { + return nil, fmt.Errorf("error get file stat. path=%s; %w", path, err) + } + + file, err := os.Open(path) if err != nil { - return nil, fmt.Errorf("error reading file_path=%s; %w", path, err) + return nil, fmt.Errorf("error opening file. path=%s; %w", path, err) } - scanner := bufio.NewScanner(content) + scanner := bufio.NewScanner(file) for scanner.Scan() { fileContents[path] = append(fileContents[path], logs.Result{ - // Id: , guess I can ignore the Id at this stage - // Time: , not sure how to reliably get the time here. This varies based on the log type. + Time: fInfo.ModTime().Format(time.RFC3339), // Labels: , all the records will have the same labels. Is it necessary to add it here? Message: strings.TrimSpace(scanner.Text()), }) From c13a28c6ee7255fef4ff90444ededf5d0092406a Mon Sep 17 00:00:00 2001 From: Aditya Date: Fri, 20 Jan 2023 16:41:57 +0545 Subject: [PATCH 3/8] Added integration test for file search --- Makefile | 6 +++- cmd/serve.go | 15 +++++++-- pkg/config.go | 11 +++++++ pkg/files/search_test.go | 71 ++++++++++++++++++++++++++++++++++++++++ samples/config-file.yaml | 7 ++++ samples/config.yaml | 9 +---- samples/nginx-access.log | 3 ++ 7 files changed, 110 insertions(+), 12 deletions(-) create mode 100644 pkg/files/search_test.go create mode 100644 samples/config-file.yaml create mode 100644 samples/nginx-access.log diff --git a/Makefile b/Makefile index 61969a6..4ecf9ed 100644 --- a/Makefile +++ b/Makefile @@ -41,4 +41,8 @@ binaries: linux darwin windows compress .PHONY: release release: binaries mkdir -p .release - cp .bin/$(NAME)* .release/ \ No newline at end of file + cp .bin/$(NAME)* .release/ + +.PHONY: integration +integration: + go test --tags=integration ./... -count=1 -v \ No newline at end of file diff --git a/cmd/serve.go b/cmd/serve.go index 3124bae..f256fb7 100644 --- a/cmd/serve.go +++ b/cmd/serve.go @@ -37,24 +37,33 @@ func runServe(cmd *cobra.Command, configFiles []string) { } } fmt.Println(logs.GlobalBackends) + + server := SetupServer(kommonsClient) + addr := "0.0.0.0:" + strconv.Itoa(httpPort) + server.Logger.Fatal(server.Start(addr)) +} + +func SetupServer(kClient *kommons.Client) *echo.Echo { e := echo.New() // Extending the context and fetching the kubeconfig client here. // For more info see: https://echo.labstack.com/guide/context/#extending-context e.Use(func(next echo.HandlerFunc) echo.HandlerFunc { return func(c echo.Context) error { cc := &api.Context{ - Kommons: kommonsClient, + Kommons: kClient, Context: c, } return next(cc) } }) + e.GET("/", func(c echo.Context) error { return c.String(http.StatusOK, "apm-hub server running") }) + e.POST("/search", pkg.Search) - addr := "0.0.0.0:" + strconv.Itoa(httpPort) - e.Logger.Fatal(e.Start(addr)) + + return e } func init() { diff --git a/pkg/config.go b/pkg/config.go index 46d0c4d..5da28b7 100644 --- a/pkg/config.go +++ b/pkg/config.go @@ -2,6 +2,7 @@ package pkg import ( "os" + "path/filepath" "github.com/flanksource/flanksource-ui/apm-hub/api/logs" "github.com/flanksource/flanksource-ui/apm-hub/pkg/files" @@ -36,6 +37,16 @@ func ParseConfig(kommonsClient *kommons.Client, configFile string) ([]logs.Searc } if len(backend.Files) != 0 { + // If the paths are not absolute, + // They should be parsed with respect to the provided config file + for i, f := range backend.Files { + for j, p := range f.Paths { + if !filepath.IsAbs(p) { + backend.Files[i].Paths[j] = filepath.Join(filepath.Dir(configFile), p) + } + } + } + backend.Backend = &files.FileSearch{ FilesBackend: backend.Files, } diff --git a/pkg/files/search_test.go b/pkg/files/search_test.go new file mode 100644 index 0000000..d497b18 --- /dev/null +++ b/pkg/files/search_test.go @@ -0,0 +1,71 @@ +//go:build integration + +package files_test + +import ( + "bufio" + "encoding/json" + "net/http" + "net/http/httptest" + "os" + "strings" + "testing" + + "github.com/flanksource/flanksource-ui/apm-hub/api/logs" + "github.com/flanksource/flanksource-ui/apm-hub/cmd" + "github.com/flanksource/flanksource-ui/apm-hub/pkg" +) + +func TestFileSearch(t *testing.T) { + confPath := "../../samples/config-file.yaml" + + backend, err := pkg.ParseConfig(nil, confPath) + if err != nil { + t.Fatal("Fail to parse the config file", err) + } + logs.GlobalBackends = append(logs.GlobalBackends, backend...) + + sp := logs.SearchParams{ + Labels: map[string]string{ + "name": "acmehost", + "type": "Nginx", + }, + } + b, err := json.Marshal(sp) + if err != nil { + t.Fatal("Fail to marshal search param") + } + + req := httptest.NewRequest(http.MethodPost, "/search", strings.NewReader(string(b))) + req.Header.Add("Content-Type", "application/json") + rec := httptest.NewRecorder() + + e := cmd.SetupServer(nil) + e.ServeHTTP(rec, req) + + var res logs.SearchResults + if err := json.NewDecoder(rec.Body).Decode(&res); err != nil { + t.Fatal("Failed to decode the search result") + } + + nginxLogFile, err := os.Open("../../samples/nginx-access.log") + if err != nil { + t.Fatal("Fail to read nginx log", err) + } + + scanner := bufio.NewScanner(nginxLogFile) + var lines []string + for scanner.Scan() { + lines = append(lines, scanner.Text()) + } + + if len(res.Results) != len(lines) { + t.Fatalf("Expected [%d] lines but got [%d]", len(lines), len(res.Results)) + } + + for i := range res.Results { + if res.Results[i].Message != lines[i] { + t.Fatalf("Incorrect line [%d]. Expected %s got %s", i+1, lines[i], res.Results[i].Message) + } + } +} diff --git a/samples/config-file.yaml b/samples/config-file.yaml new file mode 100644 index 0000000..50ffc32 --- /dev/null +++ b/samples/config-file.yaml @@ -0,0 +1,7 @@ +backends: + - file: + - labels: + name: acmehost + type: Nginx + path: + - nginx-access.log diff --git a/samples/config.yaml b/samples/config.yaml index 25f44a1..37dd7c0 100644 --- a/samples/config.yaml +++ b/samples/config.yaml @@ -1,10 +1,3 @@ backends: - kubernetes: - kubeconfig: - - - file: - - labels: - name: acmehost - type: Nginx - path: - - /var/local/cyza/nginx/log/access.log + kubeconfig: \ No newline at end of file diff --git a/samples/nginx-access.log b/samples/nginx-access.log new file mode 100644 index 0000000..c2e598f --- /dev/null +++ b/samples/nginx-access.log @@ -0,0 +1,3 @@ +127.0.0.1 - - [20/Jan/2023:10:15:30 +0000] "GET /index.html HTTP/1.1" 200 612 "-" "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4389.82 Safari/537.36" +127.0.0.1 - - [20/Jan/2023:10:15:32 +0000] "GET /about.html HTTP/1.1" 200 754 "-" "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4389.82 Safari/537.36" +127.0.0.1 - - [20/Jan/2023:10:15:34 +0000] "GET /contact.html HTTP/1.1" 200 834 "-" "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4389.82 Safari/537.36" From 3be51d63e34ced2faf8b863a15d007dd985481c5 Mon Sep 17 00:00:00 2001 From: Aditya Date: Thu, 9 Feb 2023 10:13:21 +0545 Subject: [PATCH 4/8] Don't error out on file read. - Add filepath label. - Modified integration test. --- go.mod | 5 ++++- go.sum | 9 +++++++++ pkg/files/search.go | 39 +++++++++++++++++++++++++-------------- pkg/files/search_test.go | 19 +++++++++++++------ 4 files changed, 51 insertions(+), 21 deletions(-) diff --git a/go.mod b/go.mod index 832bec1..061bdb6 100644 --- a/go.mod +++ b/go.mod @@ -6,7 +6,7 @@ require ( github.com/labstack/echo/v4 v4.6.3 github.com/spf13/cobra v1.1.1 github.com/spf13/pflag v1.0.5 - gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b + gopkg.in/yaml.v3 v3.0.1 k8s.io/api v0.20.4 k8s.io/apimachinery v0.20.4 ) @@ -98,12 +98,15 @@ require ( github.com/peterbourgon/diskv v2.0.1+incompatible // indirect github.com/pierrec/lz4 v2.3.0+incompatible // indirect github.com/pkg/errors v0.9.1 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect github.com/rainycape/unidecode v0.0.0-20150907023854-cb7f23ec59be // indirect github.com/ryanuber/go-glob v1.0.0 // indirect github.com/sergi/go-diff v1.0.0 // indirect github.com/sirupsen/logrus v1.7.0 // indirect github.com/spf13/afero v1.2.2 // indirect github.com/src-d/gcfg v1.4.0 // indirect + github.com/stretchr/objx v0.5.0 // indirect + github.com/stretchr/testify v1.8.1 // indirect github.com/ugorji/go/codec v1.1.7 // indirect github.com/ulikunitz/xz v0.5.5 // indirect github.com/valyala/bytebufferpool v1.0.0 // indirect diff --git a/go.sum b/go.sum index 3adba6f..0cde6d3 100644 --- a/go.sum +++ b/go.sum @@ -613,6 +613,9 @@ github.com/src-d/gcfg v1.4.0/go.mod h1:p/UMsR43ujA89BJY9duynAwIpvqEujIH/jFlfL7jW github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0 h1:1zr/of2m5FGMsad5YfcqgdqdWrIhu+EBEJRhR1U7z/c= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/testify v0.0.0-20161117074351-18a02ba4a312/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= @@ -621,6 +624,10 @@ github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5 github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= github.com/tidwall/gjson v1.6.7/go.mod h1:zeFuBCIqD4sN/gmqBzZ4j7Jd6UcA2Fc56x7QFsv+8fI= github.com/tidwall/match v1.0.3/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= @@ -993,6 +1000,8 @@ gopkg.in/yaml.v3 v3.0.0-20190924164351-c8b7dadae555/go.mod h1:K4uyk7z7BCEPqu6E+C gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo= gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= gotest.tools/v3 v3.0.0/go.mod h1:TUP+/YtXl/dp++T+SZ5v2zUmLVBHmptSb/ajDLCJ+3c= diff --git a/pkg/files/search.go b/pkg/files/search.go index bf30185..9eb9b7b 100644 --- a/pkg/files/search.go +++ b/pkg/files/search.go @@ -2,11 +2,11 @@ package files import ( "bufio" - "fmt" "os" "strings" "time" + "github.com/flanksource/commons/logger" "github.com/flanksource/flanksource-ui/apm-hub/api/logs" ) @@ -22,11 +22,7 @@ func (t *FileSearch) Search(q *logs.SearchParams) (r logs.SearchResults, err err continue } - files, err := readFilesLines(b.Paths) - if err != nil { - return res, fmt.Errorf("readFilesLines(); %w", err) - } - + files := readFilesLines(b.Paths, q.Labels) for _, content := range files { res.Results = append(res.Results, content...) } @@ -37,32 +33,37 @@ func (t *FileSearch) Search(q *logs.SearchParams) (r logs.SearchResults, err err type logsPerFile map[string][]logs.Result -// readFilesLines will take a list of file paths -// and then return each lines of those files. -func readFilesLines(paths []string) (logsPerFile, error) { +// readFilesLines takes a list of file paths and returns each lines of those files. +// If labels are also passed, it'll attach those labels to each lines of those files. +func readFilesLines(paths []string, labelsToAttach map[string]string) logsPerFile { fileContents := make(logsPerFile, len(paths)) for _, path := range paths { fInfo, err := os.Stat(path) if err != nil { - return nil, fmt.Errorf("error get file stat. path=%s; %w", path, err) + logger.Warnf("error get file stat. path=%s; %w", path, err) + continue } file, err := os.Open(path) if err != nil { - return nil, fmt.Errorf("error opening file. path=%s; %w", path, err) + logger.Warnf("error opening file. path=%s; %w", path, err) + continue } + // All lines of the same file will share these labels + labels := mergeMap(map[string]string{"filepath": path}, labelsToAttach) + scanner := bufio.NewScanner(file) for scanner.Scan() { fileContents[path] = append(fileContents[path], logs.Result{ - Time: fInfo.ModTime().Format(time.RFC3339), - // Labels: , all the records will have the same labels. Is it necessary to add it here? + Time: fInfo.ModTime().Format(time.RFC3339), + Labels: labels, Message: strings.TrimSpace(scanner.Text()), }) } } - return fileContents, nil + return fileContents } func matchQueryLabels(want, have map[string]string) bool { @@ -74,3 +75,13 @@ func matchQueryLabels(want, have map[string]string) bool { return true } + +// mergeMap will merge map b into a. +// On key collision, map b takes precedence. +func mergeMap(a, b map[string]string) map[string]string { + for k, v := range b { + a[k] = v + } + + return a +} diff --git a/pkg/files/search_test.go b/pkg/files/search_test.go index d497b18..b10fbf1 100644 --- a/pkg/files/search_test.go +++ b/pkg/files/search_test.go @@ -1,4 +1,4 @@ -//go:build integration +// go:build integration package files_test @@ -14,11 +14,11 @@ import ( "github.com/flanksource/flanksource-ui/apm-hub/api/logs" "github.com/flanksource/flanksource-ui/apm-hub/cmd" "github.com/flanksource/flanksource-ui/apm-hub/pkg" + "github.com/stretchr/testify/assert" ) func TestFileSearch(t *testing.T) { confPath := "../../samples/config-file.yaml" - backend, err := pkg.ParseConfig(nil, confPath) if err != nil { t.Fatal("Fail to parse the config file", err) @@ -48,7 +48,8 @@ func TestFileSearch(t *testing.T) { t.Fatal("Failed to decode the search result") } - nginxLogFile, err := os.Open("../../samples/nginx-access.log") + filePath := "../../samples/nginx-access.log" + nginxLogFile, err := os.Open(filePath) if err != nil { t.Fatal("Fail to read nginx log", err) } @@ -63,9 +64,15 @@ func TestFileSearch(t *testing.T) { t.Fatalf("Expected [%d] lines but got [%d]", len(lines), len(res.Results)) } - for i := range res.Results { - if res.Results[i].Message != lines[i] { - t.Fatalf("Incorrect line [%d]. Expected %s got %s", i+1, lines[i], res.Results[i].Message) + for i, r := range res.Results { + if r.Message != lines[i] { + t.Fatalf("Incorrect line [%d]. Expected %s got %s", i+1, lines[i], r.Message) } + + assert.Equal(t, r.Labels, map[string]string{ + "filepath": filePath, + "name": "acmehost", + "type": "Nginx", + }) } } From b0b0f72aa031523782e391e2f0121394d48b438d Mon Sep 17 00:00:00 2001 From: Aditya Date: Thu, 9 Feb 2023 10:22:01 +0545 Subject: [PATCH 5/8] Fixed go build flag. Moved to a different file. Added 1 unit test --- Makefile | 4 + pkg/files/search_integration_test.go | 78 +++++++++++++++++ pkg/files/search_test.go | 125 +++++++++++++-------------- 3 files changed, 141 insertions(+), 66 deletions(-) create mode 100644 pkg/files/search_integration_test.go diff --git a/Makefile b/Makefile index 4ecf9ed..96b279a 100644 --- a/Makefile +++ b/Makefile @@ -43,6 +43,10 @@ release: binaries mkdir -p .release cp .bin/$(NAME)* .release/ +.PHONY: test +test: + go test ./... -count=1 -v + .PHONY: integration integration: go test --tags=integration ./... -count=1 -v \ No newline at end of file diff --git a/pkg/files/search_integration_test.go b/pkg/files/search_integration_test.go new file mode 100644 index 0000000..63d8081 --- /dev/null +++ b/pkg/files/search_integration_test.go @@ -0,0 +1,78 @@ +//go:build integration + +package files_test + +import ( + "bufio" + "encoding/json" + "net/http" + "net/http/httptest" + "os" + "strings" + "testing" + + "github.com/flanksource/flanksource-ui/apm-hub/api/logs" + "github.com/flanksource/flanksource-ui/apm-hub/cmd" + "github.com/flanksource/flanksource-ui/apm-hub/pkg" + "github.com/stretchr/testify/assert" +) + +func TestFileSearch(t *testing.T) { + confPath := "../../samples/config-file.yaml" + backend, err := pkg.ParseConfig(nil, confPath) + if err != nil { + t.Fatal("Fail to parse the config file", err) + } + logs.GlobalBackends = append(logs.GlobalBackends, backend...) + + sp := logs.SearchParams{ + Labels: map[string]string{ + "name": "acmehost", + "type": "Nginx", + }, + } + b, err := json.Marshal(sp) + if err != nil { + t.Fatal("Fail to marshal search param") + } + + req := httptest.NewRequest(http.MethodPost, "/search", strings.NewReader(string(b))) + req.Header.Add("Content-Type", "application/json") + rec := httptest.NewRecorder() + + e := cmd.SetupServer(nil) + e.ServeHTTP(rec, req) + + var res logs.SearchResults + if err := json.NewDecoder(rec.Body).Decode(&res); err != nil { + t.Fatal("Failed to decode the search result") + } + + filePath := "../../samples/nginx-access.log" + nginxLogFile, err := os.Open(filePath) + if err != nil { + t.Fatal("Fail to read nginx log", err) + } + + scanner := bufio.NewScanner(nginxLogFile) + var lines []string + for scanner.Scan() { + lines = append(lines, scanner.Text()) + } + + if len(res.Results) != len(lines) { + t.Fatalf("Expected [%d] lines but got [%d]", len(lines), len(res.Results)) + } + + for i, r := range res.Results { + if r.Message != lines[i] { + t.Fatalf("Incorrect line [%d]. Expected %s got %s", i+1, lines[i], r.Message) + } + + assert.Equal(t, r.Labels, map[string]string{ + "filepath": filePath, + "name": "acmehost", + "type": "Nginx", + }) + } +} diff --git a/pkg/files/search_test.go b/pkg/files/search_test.go index b10fbf1..a5a5be0 100644 --- a/pkg/files/search_test.go +++ b/pkg/files/search_test.go @@ -1,78 +1,71 @@ -// go:build integration - -package files_test +package files import ( - "bufio" - "encoding/json" - "net/http" - "net/http/httptest" - "os" - "strings" + "reflect" "testing" - - "github.com/flanksource/flanksource-ui/apm-hub/api/logs" - "github.com/flanksource/flanksource-ui/apm-hub/cmd" - "github.com/flanksource/flanksource-ui/apm-hub/pkg" - "github.com/stretchr/testify/assert" ) -func TestFileSearch(t *testing.T) { - confPath := "../../samples/config-file.yaml" - backend, err := pkg.ParseConfig(nil, confPath) - if err != nil { - t.Fatal("Fail to parse the config file", err) +func Test_mergeMap(t *testing.T) { + type args struct { + a map[string]string + b map[string]string } - logs.GlobalBackends = append(logs.GlobalBackends, backend...) - - sp := logs.SearchParams{ - Labels: map[string]string{ - "name": "acmehost", - "type": "Nginx", + tests := []struct { + name string + args args + want map[string]string + }{ + { + name: "no overlaps", + args: args{ + a: map[string]string{"name": "flanksource"}, + b: map[string]string{"foo": "bar"}, + }, + want: map[string]string{ + "name": "flanksource", + "foo": "bar", + }, + }, + { + name: "overlaps", + args: args{ + a: map[string]string{"name": "flanksource", "foo": "baz"}, + b: map[string]string{"foo": "bar"}, + }, + want: map[string]string{ + "name": "flanksource", + "foo": "bar", + }, + }, + { + name: "overlaps II", + args: args{ + a: map[string]string{"name": "github", "foo": "baz"}, + b: map[string]string{"name": "flanksource", "foo": "bar"}, + }, + want: map[string]string{ + "name": "flanksource", + "foo": "bar", + }, + }, + { + name: "ditto", + args: args{ + a: map[string]string{"name": "flanksource", "foo": "bar"}, + b: map[string]string{"name": "flanksource", "foo": "bar"}, + }, + want: map[string]string{ + "name": "flanksource", + "foo": "bar", + }, }, } - b, err := json.Marshal(sp) - if err != nil { - t.Fatal("Fail to marshal search param") - } - - req := httptest.NewRequest(http.MethodPost, "/search", strings.NewReader(string(b))) - req.Header.Add("Content-Type", "application/json") - rec := httptest.NewRecorder() - - e := cmd.SetupServer(nil) - e.ServeHTTP(rec, req) - - var res logs.SearchResults - if err := json.NewDecoder(rec.Body).Decode(&res); err != nil { - t.Fatal("Failed to decode the search result") - } - - filePath := "../../samples/nginx-access.log" - nginxLogFile, err := os.Open(filePath) - if err != nil { - t.Fatal("Fail to read nginx log", err) - } - - scanner := bufio.NewScanner(nginxLogFile) - var lines []string - for scanner.Scan() { - lines = append(lines, scanner.Text()) - } - - if len(res.Results) != len(lines) { - t.Fatalf("Expected [%d] lines but got [%d]", len(lines), len(res.Results)) - } - - for i, r := range res.Results { - if r.Message != lines[i] { - t.Fatalf("Incorrect line [%d]. Expected %s got %s", i+1, lines[i], r.Message) - } - assert.Equal(t, r.Labels, map[string]string{ - "filepath": filePath, - "name": "acmehost", - "type": "Nginx", + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := mergeMap(tt.args.a, tt.args.b); !reflect.DeepEqual(got, tt.want) { + t.Errorf("mergeMap() = %v, want %v", got, tt.want) + } }) } } From 5fff861c7fa47a3b361e69b813108d290e4689b0 Mon Sep 17 00:00:00 2001 From: Aditya Date: Thu, 9 Feb 2023 10:23:05 +0545 Subject: [PATCH 6/8] chore: go mod tidy --- go.mod | 3 +-- go.sum | 3 --- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/go.mod b/go.mod index 061bdb6..6441483 100644 --- a/go.mod +++ b/go.mod @@ -6,6 +6,7 @@ require ( github.com/labstack/echo/v4 v4.6.3 github.com/spf13/cobra v1.1.1 github.com/spf13/pflag v1.0.5 + github.com/stretchr/testify v1.8.1 gopkg.in/yaml.v3 v3.0.1 k8s.io/api v0.20.4 k8s.io/apimachinery v0.20.4 @@ -105,8 +106,6 @@ require ( github.com/sirupsen/logrus v1.7.0 // indirect github.com/spf13/afero v1.2.2 // indirect github.com/src-d/gcfg v1.4.0 // indirect - github.com/stretchr/objx v0.5.0 // indirect - github.com/stretchr/testify v1.8.1 // indirect github.com/ugorji/go/codec v1.1.7 // indirect github.com/ulikunitz/xz v0.5.5 // indirect github.com/valyala/bytebufferpool v1.0.0 // indirect diff --git a/go.sum b/go.sum index 0cde6d3..0037e1a 100644 --- a/go.sum +++ b/go.sum @@ -614,7 +614,6 @@ github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+ github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= -github.com/stretchr/objx v0.5.0 h1:1zr/of2m5FGMsad5YfcqgdqdWrIhu+EBEJRhR1U7z/c= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/testify v0.0.0-20161117074351-18a02ba4a312/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= @@ -622,7 +621,6 @@ github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UV github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= @@ -998,7 +996,6 @@ gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v3 v3.0.0-20190924164351-c8b7dadae555/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= From b5e8c5e52d700568aa223ac7839ebc1b1226b8fa Mon Sep 17 00:00:00 2001 From: Aditya Date: Thu, 9 Feb 2023 13:24:00 +0545 Subject: [PATCH 7/8] renamed "filepath" -> "path" --- pkg/files/search.go | 2 +- pkg/files/search_integration_test.go | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/pkg/files/search.go b/pkg/files/search.go index 9eb9b7b..be871fb 100644 --- a/pkg/files/search.go +++ b/pkg/files/search.go @@ -51,7 +51,7 @@ func readFilesLines(paths []string, labelsToAttach map[string]string) logsPerFil } // All lines of the same file will share these labels - labels := mergeMap(map[string]string{"filepath": path}, labelsToAttach) + labels := mergeMap(map[string]string{"path": path}, labelsToAttach) scanner := bufio.NewScanner(file) for scanner.Scan() { diff --git a/pkg/files/search_integration_test.go b/pkg/files/search_integration_test.go index 63d8081..f125582 100644 --- a/pkg/files/search_integration_test.go +++ b/pkg/files/search_integration_test.go @@ -70,9 +70,9 @@ func TestFileSearch(t *testing.T) { } assert.Equal(t, r.Labels, map[string]string{ - "filepath": filePath, - "name": "acmehost", - "type": "Nginx", + "path": filePath, + "name": "acmehost", + "type": "Nginx", }) } } From 7d2afc72ea83a7e1e0ce30efa9f710ea276932a0 Mon Sep 17 00:00:00 2001 From: Aditya Date: Thu, 9 Feb 2023 14:06:49 +0545 Subject: [PATCH 8/8] Added support for globs --- pkg/files/search.go | 24 +++++- pkg/files/search_integration_test.go | 123 ++++++++++++++++++--------- pkg/files/search_test.go | 2 +- samples/config-file.yaml | 5 ++ samples/nginx-error.log | 3 + 5 files changed, 113 insertions(+), 44 deletions(-) create mode 100644 samples/nginx-error.log diff --git a/pkg/files/search.go b/pkg/files/search.go index be871fb..65e4c2d 100644 --- a/pkg/files/search.go +++ b/pkg/files/search.go @@ -3,6 +3,7 @@ package files import ( "bufio" "os" + "path/filepath" "strings" "time" @@ -37,7 +38,7 @@ type logsPerFile map[string][]logs.Result // If labels are also passed, it'll attach those labels to each lines of those files. func readFilesLines(paths []string, labelsToAttach map[string]string) logsPerFile { fileContents := make(logsPerFile, len(paths)) - for _, path := range paths { + for _, path := range unfoldGlobs(paths) { fInfo, err := os.Stat(path) if err != nil { logger.Warnf("error get file stat. path=%s; %w", path, err) @@ -51,7 +52,7 @@ func readFilesLines(paths []string, labelsToAttach map[string]string) logsPerFil } // All lines of the same file will share these labels - labels := mergeMap(map[string]string{"path": path}, labelsToAttach) + labels := MergeMap(map[string]string{"path": path}, labelsToAttach) scanner := bufio.NewScanner(file) for scanner.Scan() { @@ -76,12 +77,27 @@ func matchQueryLabels(want, have map[string]string) bool { return true } -// mergeMap will merge map b into a. +// MergeMap will merge map b into a. // On key collision, map b takes precedence. -func mergeMap(a, b map[string]string) map[string]string { +func MergeMap(a, b map[string]string) map[string]string { for k, v := range b { a[k] = v } return a } + +func unfoldGlobs(paths []string) []string { + unfoldedPaths := make([]string, 0, len(paths)) + for _, path := range paths { + matched, err := filepath.Glob(path) + if err != nil { + logger.Warnf("invalid glob pattern. path=%s; %w", path, err) + continue + } + + unfoldedPaths = append(unfoldedPaths, matched...) + } + + return unfoldedPaths +} diff --git a/pkg/files/search_integration_test.go b/pkg/files/search_integration_test.go index f125582..ad39c3b 100644 --- a/pkg/files/search_integration_test.go +++ b/pkg/files/search_integration_test.go @@ -8,16 +8,40 @@ import ( "net/http" "net/http/httptest" "os" + "reflect" "strings" "testing" "github.com/flanksource/flanksource-ui/apm-hub/api/logs" "github.com/flanksource/flanksource-ui/apm-hub/cmd" "github.com/flanksource/flanksource-ui/apm-hub/pkg" - "github.com/stretchr/testify/assert" + "github.com/flanksource/flanksource-ui/apm-hub/pkg/files" ) func TestFileSearch(t *testing.T) { + testData := []struct { + Name string + Labels map[string]string // Labels passed to the search + MatchFiles []string // MatchFiles contains the list of files that'll be read directly to compare against the search result. + }{ + { + Name: "simple", + Labels: map[string]string{ + "name": "acmehost", + "type": "Nginx", + }, + MatchFiles: []string{"../../samples/nginx-access.log"}, + }, + { + Name: "glob", + Labels: map[string]string{ + "name": "all", + "type": "Nginx", + }, + MatchFiles: []string{"../../samples/nginx-access.log", "../../samples/nginx-error.log"}, + }, + } + confPath := "../../samples/config-file.yaml" backend, err := pkg.ParseConfig(nil, confPath) if err != nil { @@ -25,54 +49,75 @@ func TestFileSearch(t *testing.T) { } logs.GlobalBackends = append(logs.GlobalBackends, backend...) - sp := logs.SearchParams{ - Labels: map[string]string{ - "name": "acmehost", - "type": "Nginx", - }, - } - b, err := json.Marshal(sp) - if err != nil { - t.Fatal("Fail to marshal search param") - } + for i, td := range testData { + t.Run(td.Name, func(t *testing.T) { + sp := logs.SearchParams{Labels: td.Labels} + b, err := json.Marshal(sp) + if err != nil { + t.Fatal("Failed to marshal search param") + } - req := httptest.NewRequest(http.MethodPost, "/search", strings.NewReader(string(b))) - req.Header.Add("Content-Type", "application/json") - rec := httptest.NewRecorder() + req := httptest.NewRequest(http.MethodPost, "/search", strings.NewReader(string(b))) + req.Header.Add("Content-Type", "application/json") + rec := httptest.NewRecorder() - e := cmd.SetupServer(nil) - e.ServeHTTP(rec, req) + e := cmd.SetupServer(nil) + e.ServeHTTP(rec, req) - var res logs.SearchResults - if err := json.NewDecoder(rec.Body).Decode(&res); err != nil { - t.Fatal("Failed to decode the search result") - } + var res logs.SearchResults + if err := json.NewDecoder(rec.Body).Decode(&res); err != nil { + t.Fatal("Failed to decode the search result") + } - filePath := "../../samples/nginx-access.log" - nginxLogFile, err := os.Open(filePath) - if err != nil { - t.Fatal("Fail to read nginx log", err) - } + // Directly read those files to compare it against the search result + lines := readLines(t, td.MatchFiles) - scanner := bufio.NewScanner(nginxLogFile) - var lines []string - for scanner.Scan() { - lines = append(lines, scanner.Text()) + if len(res.Results) != len(lines) { + t.Fatalf("[%d] Expected [%d] lines but got [%d]", i, len(lines), len(res.Results)) + } + + for i, r := range res.Results { + if r.Message != lines[i] { + t.Fatalf("[%d] Incorrect line [%d]. Expected %s got %s", i, i+1, lines[i], r.Message) + } + + matchLabels(t, r.Labels, td.Labels, td.MatchFiles) + } + }) } +} - if len(res.Results) != len(lines) { - t.Fatalf("Expected [%d] lines but got [%d]", len(lines), len(res.Results)) +// matchLabels tries to match the label returned from the search result +// against many labels by iterating through the given paths. +func matchLabels(t *testing.T, labels, searchLabels map[string]string, paths []string) { + t.Helper() + + for _, path := range paths { + expectedLabel := files.MergeMap(searchLabels, map[string]string{"path": path}) + if reflect.DeepEqual(labels, expectedLabel) { + return + } } - for i, r := range res.Results { - if r.Message != lines[i] { - t.Fatalf("Incorrect line [%d]. Expected %s got %s", i+1, lines[i], r.Message) + t.Fatalf("Incorrect label. Got [%v]\n", labels) +} + +// readLines is a helper func to read the file lines +func readLines(t *testing.T, paths []string) []string { + t.Helper() + + var lines []string + for _, filePath := range paths { + nginxLogFile, err := os.Open(filePath) + if err != nil { + t.Fatal("Fail to read nginx log", err) } - assert.Equal(t, r.Labels, map[string]string{ - "path": filePath, - "name": "acmehost", - "type": "Nginx", - }) + scanner := bufio.NewScanner(nginxLogFile) + for scanner.Scan() { + lines = append(lines, scanner.Text()) + } } + + return lines } diff --git a/pkg/files/search_test.go b/pkg/files/search_test.go index a5a5be0..06bfe48 100644 --- a/pkg/files/search_test.go +++ b/pkg/files/search_test.go @@ -63,7 +63,7 @@ func Test_mergeMap(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - if got := mergeMap(tt.args.a, tt.args.b); !reflect.DeepEqual(got, tt.want) { + if got := MergeMap(tt.args.a, tt.args.b); !reflect.DeepEqual(got, tt.want) { t.Errorf("mergeMap() = %v, want %v", got, tt.want) } }) diff --git a/samples/config-file.yaml b/samples/config-file.yaml index 50ffc32..319aa9d 100644 --- a/samples/config-file.yaml +++ b/samples/config-file.yaml @@ -5,3 +5,8 @@ backends: type: Nginx path: - nginx-access.log + - labels: + name: all + type: Nginx + path: + - "*.log" diff --git a/samples/nginx-error.log b/samples/nginx-error.log new file mode 100644 index 0000000..2db0a9d --- /dev/null +++ b/samples/nginx-error.log @@ -0,0 +1,3 @@ +2023/02/08 16:51:50 [notice] 1#1: signal 17 (SIGCHLD) received from 44 +2023/02/08 16:51:50 [notice] 1#1: worker process 44 exited with code 0 +2023/02/08 16:51:50 [notice] 1#1: exit \ No newline at end of file