diff --git a/.github/workflows/build.yml b/.github/workflows/test.yml similarity index 97% rename from .github/workflows/build.yml rename to .github/workflows/test.yml index d0eed00..3252a1b 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/test.yml @@ -1,4 +1,4 @@ -name: build +name: test on: pull_request: @@ -9,7 +9,7 @@ on: - '**' jobs: - build: + test: runs-on: ubuntu-latest steps: - name: Checkout diff --git a/README.md b/README.md index 6d446ca..692e734 100644 --- a/README.md +++ b/README.md @@ -1,42 +1,38 @@ -## What is ``go-watch-logs`` +

+ Go Watch Logs
+ Monitor static logs file for patterns and send alerts to MS Teams
+

-1. ``go-watch-logs`` is a standalone CLI which can tail any console or API request and logs. -2. You can configure ``go-watch-logs`` to send a notification whenever a regexp match is found on a tailed log file. -3. ``go-watch-logs`` supports sending alert notifications to MS teams and via Email. For more info, see [go-alertnotification](https://github.com/rakutentech/go-alertnotification). -4. ``go-watch-logs`` has an inbuilt throttler counter which allows you to set a limit to num of alert messages for a regexp within the duration. +**Quick Setup:** One command to install Go and manage versions. + +**Hassle Free:** Doesn't require root or sudo. + +**Platform:** Supports (arm64, arch64, Mac, Mac M1, Ubuntu and Windows). + +**Flexible:** Works with any logs file, huge to massive, log rotation is supported. + +**Notify:** Supports MS Teams. Emails, Slack (coming soon). ### Install using go ```bash -export GO111MODULE=on -go get -u github.com/rakutentech/go-watch-logs +go install github.com/rakutentech/go-watch-logs@latest go-watch-logs --help ``` -### Or Install using executable +### Install using curl Use this method if go is not installed on your server ```bash -git clone https://github.com/rakutentech/go-watch-logs/ -./bin/linux/go-watch-logs --help -``` - -## Run - -```bash -cp .env.example .env -set -a && . ./.env && set +a -go-watch-logs --watch-file=/path/to/logs.log "regexp1" "regexp2" > /dev/null 2>&1 & +curl -sL https://raw.githubusercontent.com/rakutentech/go-watch-logs/master/install.sh | sh ``` ## Run it on a cron -``go-watch-logs`` shouldn't kill itself even when the watching file is removed. It is capable of recovering itself. -But it is good idea to add it to your cronjob. ``` -* * * * * set -a && . /path/to/.env && set +a && flock -n /tmp/go-watch-logs.lock go-watch-logs --watch-file=/path/to/logs.log "regexp1" "regexp2" > /dev/null 2>&1 & +* * * * * go-watch-logs --file-path=my.log --match="error:pattern1|error:pattern2" --ms-teams-hook="https://outlook.office.com/webhook/xxxxx" ``` @@ -44,76 +40,31 @@ But it is good idea to add it to your cronjob. ## Help -``` ---help - Prints this Usage ---limit int - Limit to notify (default 10) ---seconds int - Limit notify per number of second (default 30) ---offset int - Offset Limit to ignore first few (default 0) ---watch-file string - Path to the file to tail --ignore-regexp string - One regexp to ignore reporting --ignore-regexp="donotmatch1|donotmatch2" --recovery-cmd string - Shell cmd to execute on match found (default "") - -Basic Usage: - go-watch-logs --limit=10 --seconds=30 --watch-file=/path/to/your.log "regexp1" "regexp2" -Description: - Will send max 10 notifications every 30 seconds for regexp1 matched per line in your.log file - And will send max 10 notifications every 30 seconds for regexp2 matched per line in your.log file -``` - -## Build it your self - ```sh -GOOS=linux GOARCH=amd64 go build watch_logs.go; mv watch_logs ./bin/linux/go-watch-logs -GOOS=darwin GOARCH=amd64 go build watch_logs.go; mv watch_logs ./bin/mac/go-watch-logs + -db-path string + path to db file (default "go-watch-logs.db") + -file-path string + path to logs file + -ignore string + ignore pattern + -match string + match pattern + -ms-teams-hook string + ms teams webhook + -version + print version ``` + ---- ## Performance Notes -1. Load on http server - -![load.png](./docs/load.png) - -2. ``go-watch-logs`` on the ``ssl_access_log`` for ``HTTP/1.1" 50`` errors. - ``` -go-watch-logs --limit=2 --seconds=60 --watch-file=/var/log/httpd/vhost/ssl_access_log "HTTP/1.1\" 50" ``` -3. ``go-watch-logs`` doesn't consume much memory. - -Check the RSS is the Resident Set Size and which is used to show how much memory is allocated to that process and is in RAM. VSZ is the Virtual Memory Size. It includes all memory that the process can access - -> Total memory used through out is **12720 KB** i.e. **12 MB** - -```s -Address Kbytes RSS Dirty Mode Mapping -total kB 709040 12720 6552 -``` - -> Viewing httpd app and ``go-watch-logs`` pids with top command - -```s -PID USER PR NI VIRT RES SHR S %CPU %MEM TIME+ COMMAND -108778 appuser+ 20 0 375916 10340 4864 S 1.0 0.3 0:18.91 main -108770 appuser+ 20 0 375916 10352 4884 S 1.0 0.3 0:18.88 main -125938 appuser+ 20 0 496132 2600 1884 S 0.0 0.1 0:00.99 start_server -125902 appuser+ 20 0 496132 8008 1620 S 0.0 0.2 0:01.17 start_server - 23545 appuser+ 20 0 709036 12340 3920 S 0.0 0.3 0:01.65 go-watch-logs - 45145 apache 20 0 142776 3052 640 S 0.0 0.1 0:09.71 httpd - 99944 root 20 0 142776 6944 4544 S 0.0 0.2 0:15.16 httpd -``` ## Credits 1. https://github.com/rakutentech/go-alertnotification -2. https://github.com/hpcloud/tail diff --git a/main.go b/main.go index d7169ac..397bea5 100644 --- a/main.go +++ b/main.go @@ -58,7 +58,11 @@ func watch() { fmt.Println("ms teams message:") fmt.Println(teamsMsg) alert := n.NewAlert(fmt.Errorf(teamsMsg), nil) - alert.Notify() + go func() { + if err := alert.Notify(); err != nil { + fmt.Println("error sending alert:", err) + } + }() } fmt.Printf("error count: %d\n", errorCount) @@ -67,25 +71,14 @@ func watch() { func SetupFlags() { flag.StringVar(&f.filePath, "file-path", "", "path to logs file") - flag.StringVar(&f.filePath, "f", "", "path to logs file") - - flag.StringVar(&f.dbPath, "db-path", "my.db", "path to db file") - flag.StringVar(&f.dbPath, "d", "go-watch-logs.db", "path to db file") - + flag.StringVar(&f.dbPath, "db-path", "go-watch-logs.db", "path to db file") flag.StringVar(&f.match, "match", "", "match pattern") - flag.StringVar(&f.match, "m", "", "match pattern") - flag.StringVar(&f.ignore, "ignore", "", "ignore pattern") - flag.StringVar(&f.ignore, "i", "", "ignore pattern") - flag.BoolVar(&f.version, "version", false, "print version") - flag.BoolVar(&f.version, "v", false, "print version") flag.StringVar(&f.msTeamsHook, "ms-teams-hook", "", "ms teams webhook") - flag.StringVar(&f.msTeamsHook, "mth", "", "ms teams webhook") flag.Parse() - } func SetMSTeams() { diff --git a/pkg/watcher_test.go b/pkg/watcher_test.go index 3ecc628..b014245 100644 --- a/pkg/watcher_test.go +++ b/pkg/watcher_test.go @@ -7,6 +7,10 @@ import ( "github.com/stretchr/testify/assert" ) +const ( + inMemory = ":memory:" +) + func setupTempFile(content string) (string, error) { tmpfile, err := os.CreateTemp("", "test.log") if err != nil { @@ -22,10 +26,10 @@ func setupTempFile(content string) (string, error) { } func TestNewWatcher(t *testing.T) { - dbName := ":memory:" - filePath := "test.log" - matchPattern := "error:1" - ignorePattern := "ignore" + dbName := inMemory + filePath := "test.log" // nolint: goconst + matchPattern := "error:1" // nolint: goconst + ignorePattern := "ignore" // nolint: goconst watcher, err := NewWatcher(dbName, filePath, matchPattern, ignorePattern) assert.NoError(t, err) @@ -44,9 +48,9 @@ error:1` assert.NoError(t, err) defer os.Remove(filePath) - dbName := ":memory:" - matchPattern := `error:1` - ignorePattern := `ignore` + dbName := inMemory + matchPattern := `error:1` // nolint: goconst + ignorePattern := `ignore` // nolint: goconst watcher, err := NewWatcher(dbName, filePath, matchPattern, ignorePattern) assert.NoError(t, err) @@ -60,7 +64,7 @@ error:1` } func TestSetAndGetLastLineNum(t *testing.T) { - dbName := ":memory:" + dbName := inMemory filePath := "test.log" matchPattern := "error:1" ignorePattern := "ignore" @@ -77,8 +81,8 @@ func TestSetAndGetLastLineNum(t *testing.T) { func TestLoadAndSaveState(t *testing.T) { dbName := "test.db" filePath := "test.log" - matchPattern := "error:1" - ignorePattern := "ignore" + matchPattern := "error:1" // nolint: goconst + ignorePattern := "ignore" // nolint: goconst watcher, err := NewWatcher(dbName, filePath, matchPattern, ignorePattern) assert.NoError(t, err) @@ -104,9 +108,9 @@ line2` assert.NoError(t, err) defer os.Remove(filePath) - dbName := ":memory:" - matchPattern := `error:1` - ignorePattern := `ignore` + dbName := inMemory + matchPattern := `error:1` // nolint: goconst + ignorePattern := `ignore` // nolint: goconst watcher, err := NewWatcher(dbName, filePath, matchPattern, ignorePattern) assert.NoError(t, err) @@ -119,7 +123,7 @@ line2` assert.Equal(t, "error:1", last) // Simulate log rotation by truncating the file - err = os.WriteFile(filePath, []byte("new content\nerror:1\n"), 0644) + err = os.WriteFile(filePath, []byte("new content\nerror:1\n"), 0644) // nolint: gosec assert.NoError(t, err) // Ensure Watcher detects log rotation