From 908ed1041c91fdb548f77453a88cc59166a830df Mon Sep 17 00:00:00 2001 From: Pulkit Kathuria Date: Wed, 26 Jun 2024 18:58:41 +0900 Subject: [PATCH] (sqlite) by go --- go.mod | 3 ++ go.sum | 8 +++++ main.go | 5 +-- pkg/database.go | 39 ++++++++++++++++++++++++ pkg/watcher.go | 74 ++++++++++++--------------------------------- pkg/watcher_test.go | 14 ++++----- 6 files changed, 78 insertions(+), 65 deletions(-) create mode 100644 pkg/database.go diff --git a/go.mod b/go.mod index aa3e19a..bfd5145 100644 --- a/go.mod +++ b/go.mod @@ -7,6 +7,7 @@ toolchain go1.22.3 require ( github.com/glebarez/go-sqlite v1.22.0 github.com/jasonlvhit/gocron v0.0.1 + github.com/k0kubun/pp v3.0.1+incompatible github.com/kevincobain2000/go-msteams v0.0.0-20231124044510-4369c04dd224 github.com/lmittmann/tint v1.0.4 github.com/mattn/go-isatty v0.0.20 @@ -17,7 +18,9 @@ require ( github.com/davecgh/go-spew v1.1.1 // indirect github.com/dustin/go-humanize v1.0.1 // indirect github.com/google/uuid v1.5.0 // indirect + github.com/k0kubun/colorstring v0.0.0-20150214042306-9440f1994b88 // indirect github.com/kr/pretty v0.3.1 // indirect + github.com/mattn/go-colorable v0.1.13 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect golang.org/x/sys v0.15.0 // indirect diff --git a/go.sum b/go.sum index 03f3c2f..ad208da 100644 --- a/go.sum +++ b/go.sum @@ -16,6 +16,10 @@ github.com/google/uuid v1.5.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+ github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/jasonlvhit/gocron v0.0.1 h1:qTt5qF3b3srDjeOIR4Le1LfeyvoYzJlYpqvG7tJX5YU= github.com/jasonlvhit/gocron v0.0.1/go.mod h1:k9a3TV8VcU73XZxfVHCHWMWF9SOqgoku0/QlY2yvlA4= +github.com/k0kubun/colorstring v0.0.0-20150214042306-9440f1994b88 h1:uC1QfSlInpQF+M0ao65imhwqKnz3Q2z/d8PWZRMQvDM= +github.com/k0kubun/colorstring v0.0.0-20150214042306-9440f1994b88/go.mod h1:3w7q1U84EfirKl04SVQ/s7nPm1ZPhiXd34z40TNz36k= +github.com/k0kubun/pp v3.0.1+incompatible h1:3tqvf7QgUnZ5tXO6pNAZlrvHgl6DvifjDrd9g2S9Z40= +github.com/k0kubun/pp v3.0.1+incompatible/go.mod h1:GWse8YhT0p8pT4ir3ZgBbfZild3tgzSScAn6HmfYukg= github.com/kevincobain2000/go-msteams v0.0.0-20231124044510-4369c04dd224 h1:YLj2tqX+dD34tRTW7WUuOW00G+RPBQAKGRYGNmkYuBs= github.com/kevincobain2000/go-msteams v0.0.0-20231124044510-4369c04dd224/go.mod h1:+HowoQQHg9HLfx3CYQGImGGYw20+kN9rFmUXgxrqBzo= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= @@ -27,6 +31,9 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/lmittmann/tint v1.0.4 h1:LeYihpJ9hyGvE0w+K2okPTGUdVLfng1+nDNVR4vWISc= github.com/lmittmann/tint v1.0.4/go.mod h1:HIS3gSy7qNwGCj+5oRjAutErFBl4BzdQP6cJZ0NfMwE= +github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= +github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= +github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= @@ -51,6 +58,7 @@ golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc= golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= diff --git a/main.go b/main.go index 092e65d..7fb4a71 100644 --- a/main.go +++ b/main.go @@ -23,7 +23,6 @@ type Flags struct { proxy string logLevel int msTeamsHook string - noCache bool version bool } @@ -44,7 +43,6 @@ func main() { "dbPath", f.dbPath, "min", f.min, "every", f.every, - "noCache", f.noCache, "version", f.version, "loglevel", f.logLevel, "proxy", f.proxy, @@ -102,7 +100,7 @@ func validate() { } func watch(filePath string) { - watcher, err := pkg.NewWatcher(f.dbPath, filePath, f.match, f.ignore, f.noCache) + watcher, err := pkg.NewWatcher(f.dbPath, filePath, f.match, f.ignore) if err != nil { slog.Error("Error creating watcher", "error", err.Error(), "filePath", filePath) return @@ -163,7 +161,6 @@ func flags() { flag.Uint64Var(&f.every, "every", 0, "run every n seconds (0 to run once)") flag.IntVar(&f.logLevel, "log-level", 0, "log level (0=info, 1=debug)") flag.IntVar(&f.min, "min", 1, "on minimum num of matches, it should notify") - flag.BoolVar(&f.noCache, "no-cache", false, "read back from the start of the file (default false)") flag.BoolVar(&f.version, "version", false, "") flag.StringVar(&f.proxy, "proxy", "", "http proxy for webhooks") diff --git a/pkg/database.go b/pkg/database.go new file mode 100644 index 0000000..46fe2d2 --- /dev/null +++ b/pkg/database.go @@ -0,0 +1,39 @@ +package pkg + +import ( + "database/sql" + "os" + "time" + + _ "github.com/glebarez/go-sqlite" // nolint: revive +) + +func InitDB(dbName string) (*sql.DB, error) { + db, err := sql.Open("sqlite", dbName) + if err != nil { + return nil, err + } + + _, err = db.Exec(` + CREATE TABLE IF NOT EXISTS state ( + key TEXT PRIMARY KEY, + value INTEGER + ) + `) + if err != nil { + return nil, err + } + db.SetMaxOpenConns(5) + db.SetMaxIdleConns(5) + db.SetConnMaxLifetime(time.Hour) + + return db, nil +} +func Vacuum(dbName string) error { + if _, err := os.Stat(dbName); err == nil { + if err := os.Remove(dbName); err != nil { + return err + } + } + return nil +} diff --git a/pkg/watcher.go b/pkg/watcher.go index 2055101..98c798f 100644 --- a/pkg/watcher.go +++ b/pkg/watcher.go @@ -3,22 +3,20 @@ package pkg import ( "bufio" "database/sql" - "fmt" "io" "os" "regexp" - - _ "github.com/glebarez/go-sqlite" // nolint: revive + "strings" ) type Watcher struct { db *sql.DB + dbName string // full path filePath string lastLineKey string lastFileSizeKey string matchPattern string ignorePattern string - noCache bool lastLineNum int lastFileSize int64 } @@ -28,29 +26,22 @@ func NewWatcher( filePath string, matchPattern string, ignorePattern string, - noCache bool, ) (*Watcher, error) { dbName += ".sqlite" - db, err := initDB(dbName) + db, err := InitDB(dbName) if err != nil { return nil, err } watcher := &Watcher{ db: db, + dbName: dbName, filePath: filePath, matchPattern: matchPattern, ignorePattern: ignorePattern, - noCache: noCache, - lastLineKey: Hash(filePath + "llk"), - lastFileSizeKey: Hash(filePath + "llks"), - } - if watcher.noCache { - if err := watcher.NoCache(); err != nil { - return nil, err - } + lastLineKey: "llk-" + filePath, + lastFileSizeKey: "llks-" + filePath, } - if err := watcher.loadState(); err != nil { return nil, err } @@ -58,32 +49,6 @@ func NewWatcher( return watcher, nil } -func initDB(dbName string) (*sql.DB, error) { - db, err := sql.Open("sqlite", dbName) - if err != nil { - return nil, err - } - - _, err = db.Exec(` - CREATE TABLE IF NOT EXISTS state ( - key TEXT PRIMARY KEY, - value TEXT - ) - `) - if err != nil { - return nil, err - } - return db, nil -} -func (w *Watcher) NoCache() error { - _, err := w.db.Exec(`REPLACE INTO state (key, value) VALUES (?, ?)`, w.lastLineKey, "0") - if err != nil { - return err - } - _, err = w.db.Exec(`REPLACE INTO state (key, value) VALUES (?, ?)`, w.lastFileSizeKey, "0") - return err -} - type ScanResult struct { ErrorCount int FirstLine string @@ -103,7 +68,7 @@ func (w *Watcher) Scan() (*ScanResult, error) { currentFileSize := fileInfo.Size() // Detect log rotation - if currentFileSize < w.lastFileSize { + if currentFileSize+2 < w.lastFileSize { w.lastFileSize = 0 } @@ -153,6 +118,11 @@ func (w *Watcher) Scan() (*ScanResult, error) { w.lastLineNum = currentLineNum w.lastFileSize = bytesRead if err := w.saveState(); err != nil { + if strings.HasPrefix(err.Error(), "database is locked") { + if err := Vacuum(w.dbName); err != nil { + return nil, err + } + } return nil, err } return &ScanResult{ @@ -164,33 +134,29 @@ func (w *Watcher) Scan() (*ScanResult, error) { func (w *Watcher) loadState() error { row := w.db.QueryRow(`SELECT value FROM state WHERE key = ?`, w.lastLineKey) - var lastLineStr string - err := row.Scan(&lastLineStr) + var lastLineNum int + err := row.Scan(&lastLineNum) if err != nil && err != sql.ErrNoRows { return err } - if lastLineStr != "" { - fmt.Sscanf(lastLineStr, "%d", &w.lastLineNum) // nolint: errcheck - } + w.lastLineNum = lastLineNum row = w.db.QueryRow(`SELECT value FROM state WHERE key = ?`, w.lastFileSizeKey) - var lastFileSizeStr string - err = row.Scan(&lastFileSizeStr) + var lastFileSize int64 + err = row.Scan(&lastFileSize) if err != nil && err != sql.ErrNoRows { return err } - if lastFileSizeStr != "" { - fmt.Sscanf(lastFileSizeStr, "%d", &w.lastFileSize) // nolint: errcheck - } + w.lastFileSize = lastFileSize return nil } func (w *Watcher) saveState() error { - _, err := w.db.Exec(`REPLACE INTO state (key, value) VALUES (?, ?)`, w.lastLineKey, fmt.Sprintf("%d", w.lastLineNum)) + _, err := w.db.Exec(`REPLACE INTO state (key, value) VALUES (?, ?)`, w.lastLineKey, w.lastLineNum) if err != nil { return err } - _, err = w.db.Exec(`REPLACE INTO state (key, value) VALUES (?, ?)`, w.lastFileSizeKey, fmt.Sprintf("%d", w.lastFileSize)) + _, err = w.db.Exec(`REPLACE INTO state (key, value) VALUES (?, ?)`, w.lastFileSizeKey, w.lastFileSize) return err } diff --git a/pkg/watcher_test.go b/pkg/watcher_test.go index ea8cc0a..1a9fc92 100644 --- a/pkg/watcher_test.go +++ b/pkg/watcher_test.go @@ -31,7 +31,7 @@ func TestNewWatcher(t *testing.T) { matchPattern := "error:1" // nolint: goconst ignorePattern := "ignore" // nolint: goconst - watcher, err := NewWatcher(dbName, filePath, matchPattern, ignorePattern, false) + watcher, err := NewWatcher(dbName, filePath, matchPattern, ignorePattern) assert.NoError(t, err) assert.NotNil(t, watcher) @@ -52,7 +52,7 @@ error:1` matchPattern := `error:1` // nolint: goconst ignorePattern := `ignore` // nolint: goconst - watcher, err := NewWatcher(dbName, filePath, matchPattern, ignorePattern, false) + watcher, err := NewWatcher(dbName, filePath, matchPattern, ignorePattern) assert.NoError(t, err) defer watcher.Close() @@ -75,7 +75,7 @@ line2` matchPattern := `error:1` // nolint: goconst ignorePattern := `ignore` // nolint: goconst - watcher, err := NewWatcher(dbName, filePath, matchPattern, ignorePattern, false) + watcher, err := NewWatcher(dbName, filePath, matchPattern, ignorePattern) assert.NoError(t, err) defer watcher.Close() @@ -112,7 +112,7 @@ error:1` matchPattern := `error:1` ignorePattern := `ignore` - watcher, err := NewWatcher(dbName, filePath, matchPattern, ignorePattern, false) + watcher, err := NewWatcher(dbName, filePath, matchPattern, ignorePattern) if err != nil { b.Fatal(err) } @@ -132,7 +132,7 @@ func BenchmarkLoadAndSaveState(b *testing.B) { matchPattern := "error:1" ignorePattern := "ignore" - watcher, err := NewWatcher(dbName, filePath, matchPattern, ignorePattern, false) + watcher, err := NewWatcher(dbName, filePath, matchPattern, ignorePattern) if err != nil { b.Fatal(err) } @@ -141,7 +141,7 @@ func BenchmarkLoadAndSaveState(b *testing.B) { watcher.lastLineNum = 10 for i := 0; i < b.N; i++ { - _, err := NewWatcher(dbName, filePath, matchPattern, ignorePattern, false) + _, err := NewWatcher(dbName, filePath, matchPattern, ignorePattern) if err != nil { b.Fatal(err) } @@ -162,7 +162,7 @@ line2` matchPattern := `error:1` ignorePattern := `ignore` - watcher, err := NewWatcher(dbName, filePath, matchPattern, ignorePattern, false) + watcher, err := NewWatcher(dbName, filePath, matchPattern, ignorePattern) if err != nil { b.Fatal(err) }