diff --git a/Makefile b/Makefile index 61969a6..96b279a 100644 --- a/Makefile +++ b/Makefile @@ -41,4 +41,12 @@ binaries: linux darwin windows compress .PHONY: release release: binaries mkdir -p .release - cp .bin/$(NAME)* .release/ \ No newline at end of file + cp .bin/$(NAME)* .release/ + +.PHONY: test +test: + go test ./... -count=1 -v + +.PHONY: integration +integration: + go test --tags=integration ./... -count=1 -v \ No newline at end of file diff --git a/api/logs/logs.go b/api/logs/logs.go index 69ea31d..7ff5a52 100644 --- a/api/logs/logs.go +++ b/api/logs/logs.go @@ -19,6 +19,7 @@ type SearchBackend struct { Routes []SearchRoute `json:"routes,omitempty"` Backend SearchAPI `json:"-"` Kubernetes *KubernetesSearchBackend `json:"kubernetes,omitempty"` + Files []FileSearchBackend `json:"file,omitempty" yaml:"file,omitempty"` } type SearchRoute struct { @@ -34,6 +35,11 @@ type KubernetesSearchBackend struct { Namespace string `json:"namespace,omitempty"` } +type FileSearchBackend struct { + Labels map[string]string `yaml:"labels,omitempty"` + Paths []string `yaml:"path,omitempty"` +} + type SearchParams struct { // Limit is the maximum number of results to return. Limit int64 `json:"limit,omitempty"` diff --git a/cmd/serve.go b/cmd/serve.go index 3124bae..f256fb7 100644 --- a/cmd/serve.go +++ b/cmd/serve.go @@ -37,24 +37,33 @@ func runServe(cmd *cobra.Command, configFiles []string) { } } fmt.Println(logs.GlobalBackends) + + server := SetupServer(kommonsClient) + addr := "0.0.0.0:" + strconv.Itoa(httpPort) + server.Logger.Fatal(server.Start(addr)) +} + +func SetupServer(kClient *kommons.Client) *echo.Echo { e := echo.New() // Extending the context and fetching the kubeconfig client here. // For more info see: https://echo.labstack.com/guide/context/#extending-context e.Use(func(next echo.HandlerFunc) echo.HandlerFunc { return func(c echo.Context) error { cc := &api.Context{ - Kommons: kommonsClient, + Kommons: kClient, Context: c, } return next(cc) } }) + e.GET("/", func(c echo.Context) error { return c.String(http.StatusOK, "apm-hub server running") }) + e.POST("/search", pkg.Search) - addr := "0.0.0.0:" + strconv.Itoa(httpPort) - e.Logger.Fatal(e.Start(addr)) + + return e } func init() { diff --git a/go.mod b/go.mod index 832bec1..6441483 100644 --- a/go.mod +++ b/go.mod @@ -6,7 +6,8 @@ require ( github.com/labstack/echo/v4 v4.6.3 github.com/spf13/cobra v1.1.1 github.com/spf13/pflag v1.0.5 - gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b + github.com/stretchr/testify v1.8.1 + gopkg.in/yaml.v3 v3.0.1 k8s.io/api v0.20.4 k8s.io/apimachinery v0.20.4 ) @@ -98,6 +99,7 @@ require ( github.com/peterbourgon/diskv v2.0.1+incompatible // indirect github.com/pierrec/lz4 v2.3.0+incompatible // indirect github.com/pkg/errors v0.9.1 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect github.com/rainycape/unidecode v0.0.0-20150907023854-cb7f23ec59be // indirect github.com/ryanuber/go-glob v1.0.0 // indirect github.com/sergi/go-diff v1.0.0 // indirect diff --git a/go.sum b/go.sum index 3adba6f..0037e1a 100644 --- a/go.sum +++ b/go.sum @@ -613,14 +613,19 @@ github.com/src-d/gcfg v1.4.0/go.mod h1:p/UMsR43ujA89BJY9duynAwIpvqEujIH/jFlfL7jW github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/testify v0.0.0-20161117074351-18a02ba4a312/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= github.com/tidwall/gjson v1.6.7/go.mod h1:zeFuBCIqD4sN/gmqBzZ4j7Jd6UcA2Fc56x7QFsv+8fI= github.com/tidwall/match v1.0.3/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= @@ -991,8 +996,9 @@ gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v3 v3.0.0-20190924164351-c8b7dadae555/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo= gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= gotest.tools/v3 v3.0.0/go.mod h1:TUP+/YtXl/dp++T+SZ5v2zUmLVBHmptSb/ajDLCJ+3c= diff --git a/pkg/config.go b/pkg/config.go index ed09875..5da28b7 100644 --- a/pkg/config.go +++ b/pkg/config.go @@ -1,26 +1,29 @@ package pkg import ( - "io/ioutil" + "os" + "path/filepath" "github.com/flanksource/flanksource-ui/apm-hub/api/logs" + "github.com/flanksource/flanksource-ui/apm-hub/pkg/files" k8s "github.com/flanksource/flanksource-ui/apm-hub/pkg/kubernetes" "github.com/flanksource/kommons" "gopkg.in/yaml.v3" ) -//Initilize all the backends mentioned in the config +// Initilize all the backends mentioned in the config // GlobalBackends, error func ParseConfig(kommonsClient *kommons.Client, configFile string) ([]logs.SearchBackend, error) { searchConfig := &logs.SearchConfig{} backends := []logs.SearchBackend{} - data, err := ioutil.ReadFile(configFile) + data, err := os.ReadFile(configFile) if err != nil { return nil, err } if err := yaml.Unmarshal(data, searchConfig); err != nil { return nil, err } + for _, backend := range searchConfig.Backends { if backend.Kubernetes != nil { client, err := k8s.GetKubeClient(kommonsClient, backend.Kubernetes) @@ -30,8 +33,26 @@ func ParseConfig(kommonsClient *kommons.Client, configFile string) ([]logs.Searc backend.Backend = &k8s.KubernetesSearch{ Client: client, } + backends = append(backends, backend) + } + + if len(backend.Files) != 0 { + // If the paths are not absolute, + // They should be parsed with respect to the provided config file + for i, f := range backend.Files { + for j, p := range f.Paths { + if !filepath.IsAbs(p) { + backend.Files[i].Paths[j] = filepath.Join(filepath.Dir(configFile), p) + } + } + } + + backend.Backend = &files.FileSearch{ + FilesBackend: backend.Files, + } + backends = append(backends, backend) } - backends = append(backends, backend) } + return backends, nil } diff --git a/pkg/files/search.go b/pkg/files/search.go new file mode 100644 index 0000000..65e4c2d --- /dev/null +++ b/pkg/files/search.go @@ -0,0 +1,103 @@ +package files + +import ( + "bufio" + "os" + "path/filepath" + "strings" + "time" + + "github.com/flanksource/commons/logger" + "github.com/flanksource/flanksource-ui/apm-hub/api/logs" +) + +type FileSearch struct { + FilesBackend []logs.FileSearchBackend +} + +func (t *FileSearch) Search(q *logs.SearchParams) (r logs.SearchResults, err error) { + var res logs.SearchResults + + for _, b := range t.FilesBackend { + if !matchQueryLabels(q.Labels, b.Labels) { + continue + } + + files := readFilesLines(b.Paths, q.Labels) + for _, content := range files { + res.Results = append(res.Results, content...) + } + } + + return res, nil +} + +type logsPerFile map[string][]logs.Result + +// readFilesLines takes a list of file paths and returns each lines of those files. +// If labels are also passed, it'll attach those labels to each lines of those files. +func readFilesLines(paths []string, labelsToAttach map[string]string) logsPerFile { + fileContents := make(logsPerFile, len(paths)) + for _, path := range unfoldGlobs(paths) { + fInfo, err := os.Stat(path) + if err != nil { + logger.Warnf("error get file stat. path=%s; %w", path, err) + continue + } + + file, err := os.Open(path) + if err != nil { + logger.Warnf("error opening file. path=%s; %w", path, err) + continue + } + + // All lines of the same file will share these labels + labels := MergeMap(map[string]string{"path": path}, labelsToAttach) + + scanner := bufio.NewScanner(file) + for scanner.Scan() { + fileContents[path] = append(fileContents[path], logs.Result{ + Time: fInfo.ModTime().Format(time.RFC3339), + Labels: labels, + Message: strings.TrimSpace(scanner.Text()), + }) + } + } + + return fileContents +} + +func matchQueryLabels(want, have map[string]string) bool { + for label, val := range want { + if val != have[label] { + return false + } + } + + return true +} + +// MergeMap will merge map b into a. +// On key collision, map b takes precedence. +func MergeMap(a, b map[string]string) map[string]string { + for k, v := range b { + a[k] = v + } + + return a +} + +func unfoldGlobs(paths []string) []string { + unfoldedPaths := make([]string, 0, len(paths)) + for _, path := range paths { + matched, err := filepath.Glob(path) + if err != nil { + logger.Warnf("invalid glob pattern. path=%s; %w", path, err) + continue + } + + unfoldedPaths = append(unfoldedPaths, matched...) + } + + return unfoldedPaths +} diff --git a/pkg/files/search_integration_test.go b/pkg/files/search_integration_test.go new file mode 100644 index 0000000..ad39c3b --- /dev/null +++ b/pkg/files/search_integration_test.go @@ -0,0 +1,123 @@ +//go:build integration + +package files_test + +import ( + "bufio" + "encoding/json" + "net/http" + "net/http/httptest" + "os" + "reflect" + "strings" + "testing" + + "github.com/flanksource/flanksource-ui/apm-hub/api/logs" + "github.com/flanksource/flanksource-ui/apm-hub/cmd" + "github.com/flanksource/flanksource-ui/apm-hub/pkg" + "github.com/flanksource/flanksource-ui/apm-hub/pkg/files" +) + +func TestFileSearch(t *testing.T) { + testData := []struct { + Name string + Labels map[string]string // Labels passed to the search + MatchFiles []string // MatchFiles contains the list of files that'll be read directly to compare against the search result. + }{ + { + Name: "simple", + Labels: map[string]string{ + "name": "acmehost", + "type": "Nginx", + }, + MatchFiles: []string{"../../samples/nginx-access.log"}, + }, + { + Name: "glob", + Labels: map[string]string{ + "name": "all", + "type": "Nginx", + }, + MatchFiles: []string{"../../samples/nginx-access.log", "../../samples/nginx-error.log"}, + }, + } + + confPath := "../../samples/config-file.yaml" + backend, err := pkg.ParseConfig(nil, confPath) + if err != nil { + t.Fatal("Fail to parse the config file", err) + } + logs.GlobalBackends = append(logs.GlobalBackends, backend...) + + for i, td := range testData { + t.Run(td.Name, func(t *testing.T) { + sp := logs.SearchParams{Labels: td.Labels} + b, err := json.Marshal(sp) + if err != nil { + t.Fatal("Failed to marshal search param") + } + + req := httptest.NewRequest(http.MethodPost, "/search", strings.NewReader(string(b))) + req.Header.Add("Content-Type", "application/json") + rec := httptest.NewRecorder() + + e := cmd.SetupServer(nil) + e.ServeHTTP(rec, req) + + var res logs.SearchResults + if err := json.NewDecoder(rec.Body).Decode(&res); err != nil { + t.Fatal("Failed to decode the search result") + } + + // Directly read those files to compare it against the search result + lines := readLines(t, td.MatchFiles) + + if len(res.Results) != len(lines) { + t.Fatalf("[%d] Expected [%d] lines but got [%d]", i, len(lines), len(res.Results)) + } + + for i, r := range res.Results { + if r.Message != lines[i] { + t.Fatalf("[%d] Incorrect line [%d]. Expected %s got %s", i, i+1, lines[i], r.Message) + } + + matchLabels(t, r.Labels, td.Labels, td.MatchFiles) + } + }) + } +} + +// matchLabels tries to match the label returned from the search result +// against many labels by iterating through the given paths. +func matchLabels(t *testing.T, labels, searchLabels map[string]string, paths []string) { + t.Helper() + + for _, path := range paths { + expectedLabel := files.MergeMap(searchLabels, map[string]string{"path": path}) + if reflect.DeepEqual(labels, expectedLabel) { + return + } + } + + t.Fatalf("Incorrect label. Got [%v]\n", labels) +} + +// readLines is a helper func to read the file lines +func readLines(t *testing.T, paths []string) []string { + t.Helper() + + var lines []string + for _, filePath := range paths { + nginxLogFile, err := os.Open(filePath) + if err != nil { + t.Fatal("Fail to read nginx log", err) + } + + scanner := bufio.NewScanner(nginxLogFile) + for scanner.Scan() { + lines = append(lines, scanner.Text()) + } + } + + return lines +} diff --git a/pkg/files/search_test.go b/pkg/files/search_test.go new file mode 100644 index 0000000..06bfe48 --- /dev/null +++ b/pkg/files/search_test.go @@ -0,0 +1,71 @@ +package files + +import ( + "reflect" + "testing" +) + +func Test_mergeMap(t *testing.T) { + type args struct { + a map[string]string + b map[string]string + } + tests := []struct { + name string + args args + want map[string]string + }{ + { + name: "no overlaps", + args: args{ + a: map[string]string{"name": "flanksource"}, + b: map[string]string{"foo": "bar"}, + }, + want: map[string]string{ + "name": "flanksource", + "foo": "bar", + }, + }, + { + name: "overlaps", + args: args{ + a: map[string]string{"name": "flanksource", "foo": "baz"}, + b: map[string]string{"foo": "bar"}, + }, + want: map[string]string{ + "name": "flanksource", + "foo": "bar", + }, + }, + { + name: "overlaps II", + args: args{ + a: map[string]string{"name": "github", "foo": "baz"}, + b: map[string]string{"name": "flanksource", "foo": "bar"}, + }, + want: map[string]string{ + "name": "flanksource", + "foo": "bar", + }, + }, + { + name: "ditto", + args: args{ + a: map[string]string{"name": "flanksource", "foo": "bar"}, + b: map[string]string{"name": "flanksource", "foo": "bar"}, + }, + want: map[string]string{ + "name": "flanksource", + "foo": "bar", + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := MergeMap(tt.args.a, tt.args.b); !reflect.DeepEqual(got, tt.want) { + t.Errorf("mergeMap() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/samples/config-file.yaml b/samples/config-file.yaml new file mode 100644 index 0000000..319aa9d --- /dev/null +++ b/samples/config-file.yaml @@ -0,0 +1,12 @@ +backends: + - file: + - labels: + name: acmehost + type: Nginx + path: + - nginx-access.log + - labels: + name: all + type: Nginx + path: + - "*.log" diff --git a/samples/nginx-access.log b/samples/nginx-access.log new file mode 100644 index 0000000..c2e598f --- /dev/null +++ b/samples/nginx-access.log @@ -0,0 +1,3 @@ +127.0.0.1 - - [20/Jan/2023:10:15:30 +0000] "GET /index.html HTTP/1.1" 200 612 "-" "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4389.82 Safari/537.36" +127.0.0.1 - - [20/Jan/2023:10:15:32 +0000] "GET /about.html HTTP/1.1" 200 754 "-" "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4389.82 Safari/537.36" +127.0.0.1 - - [20/Jan/2023:10:15:34 +0000] "GET /contact.html HTTP/1.1" 200 834 "-" "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4389.82 Safari/537.36" diff --git a/samples/nginx-error.log b/samples/nginx-error.log new file mode 100644 index 0000000..2db0a9d --- /dev/null +++ b/samples/nginx-error.log @@ -0,0 +1,3 @@ +2023/02/08 16:51:50 [notice] 1#1: signal 17 (SIGCHLD) received from 44 +2023/02/08 16:51:50 [notice] 1#1: worker process 44 exited with code 0 +2023/02/08 16:51:50 [notice] 1#1: exit \ No newline at end of file