(sqlite) by go

rakutentech · Jun 26, 2024 · 908ed10 · 908ed10
1 parent 2895c64
commit 908ed10
Show file tree

Hide file tree

Showing 6 changed files with 78 additions and 65 deletions.
diff --git a/go.mod b/go.mod
@@ -7,6 +7,7 @@ toolchain go1.22.3
 require (
 	github.com/glebarez/go-sqlite v1.22.0
 	github.com/jasonlvhit/gocron v0.0.1
+	github.com/k0kubun/pp v3.0.1+incompatible
 	github.com/kevincobain2000/go-msteams v0.0.0-20231124044510-4369c04dd224
 	github.com/lmittmann/tint v1.0.4
 	github.com/mattn/go-isatty v0.0.20
@@ -17,7 +18,9 @@ require (
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/dustin/go-humanize v1.0.1 // indirect
 	github.com/google/uuid v1.5.0 // indirect
+	github.com/k0kubun/colorstring v0.0.0-20150214042306-9440f1994b88 // indirect
 	github.com/kr/pretty v0.3.1 // indirect
+	github.com/mattn/go-colorable v0.1.13 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
 	golang.org/x/sys v0.15.0 // indirect

diff --git a/go.sum b/go.sum
@@ -16,6 +16,10 @@ github.com/google/uuid v1.5.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+
 github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
 github.com/jasonlvhit/gocron v0.0.1 h1:qTt5qF3b3srDjeOIR4Le1LfeyvoYzJlYpqvG7tJX5YU=
 github.com/jasonlvhit/gocron v0.0.1/go.mod h1:k9a3TV8VcU73XZxfVHCHWMWF9SOqgoku0/QlY2yvlA4=
+github.com/k0kubun/colorstring v0.0.0-20150214042306-9440f1994b88 h1:uC1QfSlInpQF+M0ao65imhwqKnz3Q2z/d8PWZRMQvDM=
+github.com/k0kubun/colorstring v0.0.0-20150214042306-9440f1994b88/go.mod h1:3w7q1U84EfirKl04SVQ/s7nPm1ZPhiXd34z40TNz36k=
+github.com/k0kubun/pp v3.0.1+incompatible h1:3tqvf7QgUnZ5tXO6pNAZlrvHgl6DvifjDrd9g2S9Z40=
+github.com/k0kubun/pp v3.0.1+incompatible/go.mod h1:GWse8YhT0p8pT4ir3ZgBbfZild3tgzSScAn6HmfYukg=
 github.com/kevincobain2000/go-msteams v0.0.0-20231124044510-4369c04dd224 h1:YLj2tqX+dD34tRTW7WUuOW00G+RPBQAKGRYGNmkYuBs=
 github.com/kevincobain2000/go-msteams v0.0.0-20231124044510-4369c04dd224/go.mod h1:+HowoQQHg9HLfx3CYQGImGGYw20+kN9rFmUXgxrqBzo=
 github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
@@ -27,6 +31,9 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
 github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
 github.com/lmittmann/tint v1.0.4 h1:LeYihpJ9hyGvE0w+K2okPTGUdVLfng1+nDNVR4vWISc=
 github.com/lmittmann/tint v1.0.4/go.mod h1:HIS3gSy7qNwGCj+5oRjAutErFBl4BzdQP6cJZ0NfMwE=
+github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA=
+github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg=
+github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
 github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
 github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
 github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
@@ -51,6 +58,7 @@ golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJ
 golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc=
 golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=

diff --git a/main.go b/main.go
@@ -23,7 +23,6 @@ type Flags struct {
 	proxy       string
 	logLevel    int
 	msTeamsHook string
-	noCache     bool
 	version     bool
 }
 
@@ -44,7 +43,6 @@ func main() {
 		"dbPath", f.dbPath,
 		"min", f.min,
 		"every", f.every,
-		"noCache", f.noCache,
 		"version", f.version,
 		"loglevel", f.logLevel,
 		"proxy", f.proxy,
@@ -102,7 +100,7 @@ func validate() {
 }
 
 func watch(filePath string) {
-	watcher, err := pkg.NewWatcher(f.dbPath, filePath, f.match, f.ignore, f.noCache)
+	watcher, err := pkg.NewWatcher(f.dbPath, filePath, f.match, f.ignore)
 	if err != nil {
 		slog.Error("Error creating watcher", "error", err.Error(), "filePath", filePath)
 		return
@@ -163,7 +161,6 @@ func flags() {
 	flag.Uint64Var(&f.every, "every", 0, "run every n seconds (0 to run once)")
 	flag.IntVar(&f.logLevel, "log-level", 0, "log level (0=info, 1=debug)")
 	flag.IntVar(&f.min, "min", 1, "on minimum num of matches, it should notify")
-	flag.BoolVar(&f.noCache, "no-cache", false, "read back from the start of the file (default false)")
 	flag.BoolVar(&f.version, "version", false, "")
 
 	flag.StringVar(&f.proxy, "proxy", "", "http proxy for webhooks")

diff --git a/pkg/database.go b/pkg/database.go
@@ -0,0 +1,39 @@
+package pkg
+
+import (
+	"database/sql"
+	"os"
+	"time"
+
+	_ "github.com/glebarez/go-sqlite" // nolint: revive
+)
+
+func InitDB(dbName string) (*sql.DB, error) {
+	db, err := sql.Open("sqlite", dbName)
+	if err != nil {
+		return nil, err
+	}
+
+	_, err = db.Exec(`
+		CREATE TABLE IF NOT EXISTS state (
+			key TEXT PRIMARY KEY,
+			value INTEGER
+		)
+	`)
+	if err != nil {
+		return nil, err
+	}
+	db.SetMaxOpenConns(5)
+	db.SetMaxIdleConns(5)
+	db.SetConnMaxLifetime(time.Hour)
+
+	return db, nil
+}
+func Vacuum(dbName string) error {
+	if _, err := os.Stat(dbName); err == nil {
+		if err := os.Remove(dbName); err != nil {
+			return err
+		}
+	}
+	return nil
+}
diff --git a/pkg/watcher.go b/pkg/watcher.go
@@ -3,22 +3,20 @@ package pkg
 import (
 	"bufio"
 	"database/sql"
-	"fmt"
 	"io"
 	"os"
 	"regexp"
-
-	_ "github.com/glebarez/go-sqlite" // nolint: revive
+	"strings"
 )
 
 type Watcher struct {
 	db              *sql.DB
+	dbName          string // full path
 	filePath        string
 	lastLineKey     string
 	lastFileSizeKey string
 	matchPattern    string
 	ignorePattern   string
-	noCache         bool
 	lastLineNum     int
 	lastFileSize    int64
 }
@@ -28,62 +26,29 @@ func NewWatcher(
 	filePath string,
 	matchPattern string,
 	ignorePattern string,
-	noCache bool,
 ) (*Watcher, error) {
 	dbName += ".sqlite"
-	db, err := initDB(dbName)
+	db, err := InitDB(dbName)
 	if err != nil {
 		return nil, err
 	}
 
 	watcher := &Watcher{
 		db:              db,
+		dbName:          dbName,
 		filePath:        filePath,
 		matchPattern:    matchPattern,
 		ignorePattern:   ignorePattern,
-		noCache:         noCache,
-		lastLineKey:     Hash(filePath + "llk"),
-		lastFileSizeKey: Hash(filePath + "llks"),
-	}
-	if watcher.noCache {
-		if err := watcher.NoCache(); err != nil {
-			return nil, err
-		}
+		lastLineKey:     "llk-" + filePath,
+		lastFileSizeKey: "llks-" + filePath,
 	}
-
 	if err := watcher.loadState(); err != nil {
 		return nil, err
 	}
 
 	return watcher, nil
 }
 
-func initDB(dbName string) (*sql.DB, error) {
-	db, err := sql.Open("sqlite", dbName)
-	if err != nil {
-		return nil, err
-	}
-
-	_, err = db.Exec(`
-		CREATE TABLE IF NOT EXISTS state (
-			key TEXT PRIMARY KEY,
-			value TEXT
-		)
-	`)
-	if err != nil {
-		return nil, err
-	}
-	return db, nil
-}
-func (w *Watcher) NoCache() error {
-	_, err := w.db.Exec(`REPLACE INTO state (key, value) VALUES (?, ?)`, w.lastLineKey, "0")
-	if err != nil {
-		return err
-	}
-	_, err = w.db.Exec(`REPLACE INTO state (key, value) VALUES (?, ?)`, w.lastFileSizeKey, "0")
-	return err
-}
-
 type ScanResult struct {
 	ErrorCount int
 	FirstLine  string
@@ -103,7 +68,7 @@ func (w *Watcher) Scan() (*ScanResult, error) {
 	currentFileSize := fileInfo.Size()
 
 	// Detect log rotation
-	if currentFileSize < w.lastFileSize {
+	if currentFileSize+2 < w.lastFileSize {
 		w.lastFileSize = 0
 	}
 
@@ -153,6 +118,11 @@ func (w *Watcher) Scan() (*ScanResult, error) {
 	w.lastLineNum = currentLineNum
 	w.lastFileSize = bytesRead
 	if err := w.saveState(); err != nil {
+		if strings.HasPrefix(err.Error(), "database is locked") {
+			if err := Vacuum(w.dbName); err != nil {
+				return nil, err
+			}
+		}
 		return nil, err
 	}
 	return &ScanResult{
@@ -164,33 +134,29 @@ func (w *Watcher) Scan() (*ScanResult, error) {
 
 func (w *Watcher) loadState() error {
 	row := w.db.QueryRow(`SELECT value FROM state WHERE key = ?`, w.lastLineKey)
-	var lastLineStr string
-	err := row.Scan(&lastLineStr)
+	var lastLineNum int
+	err := row.Scan(&lastLineNum)
 	if err != nil && err != sql.ErrNoRows {
 		return err
 	}
-	if lastLineStr != "" {
-		fmt.Sscanf(lastLineStr, "%d", &w.lastLineNum) // nolint: errcheck
-	}
+	w.lastLineNum = lastLineNum
 
 	row = w.db.QueryRow(`SELECT value FROM state WHERE key = ?`, w.lastFileSizeKey)
-	var lastFileSizeStr string
-	err = row.Scan(&lastFileSizeStr)
+	var lastFileSize int64
+	err = row.Scan(&lastFileSize)
 	if err != nil && err != sql.ErrNoRows {
 		return err
 	}
-	if lastFileSizeStr != "" {
-		fmt.Sscanf(lastFileSizeStr, "%d", &w.lastFileSize) // nolint: errcheck
-	}
+	w.lastFileSize = lastFileSize
 	return nil
 }
 
 func (w *Watcher) saveState() error {
-	_, err := w.db.Exec(`REPLACE INTO state (key, value) VALUES (?, ?)`, w.lastLineKey, fmt.Sprintf("%d", w.lastLineNum))
+	_, err := w.db.Exec(`REPLACE INTO state (key, value) VALUES (?, ?)`, w.lastLineKey, w.lastLineNum)
 	if err != nil {
 		return err
 	}
-	_, err = w.db.Exec(`REPLACE INTO state (key, value) VALUES (?, ?)`, w.lastFileSizeKey, fmt.Sprintf("%d", w.lastFileSize))
+	_, err = w.db.Exec(`REPLACE INTO state (key, value) VALUES (?, ?)`, w.lastFileSizeKey, w.lastFileSize)
 	return err
 }
 

diff --git a/pkg/watcher_test.go b/pkg/watcher_test.go
@@ -31,7 +31,7 @@ func TestNewWatcher(t *testing.T) {
 	matchPattern := "error:1" // nolint: goconst
 	ignorePattern := "ignore" // nolint: goconst
 
-	watcher, err := NewWatcher(dbName, filePath, matchPattern, ignorePattern, false)
+	watcher, err := NewWatcher(dbName, filePath, matchPattern, ignorePattern)
 	assert.NoError(t, err)
 	assert.NotNil(t, watcher)
 
@@ -52,7 +52,7 @@ error:1`
 	matchPattern := `error:1` // nolint: goconst
 	ignorePattern := `ignore` // nolint: goconst
 
-	watcher, err := NewWatcher(dbName, filePath, matchPattern, ignorePattern, false)
+	watcher, err := NewWatcher(dbName, filePath, matchPattern, ignorePattern)
 	assert.NoError(t, err)
 	defer watcher.Close()
 
@@ -75,7 +75,7 @@ line2`
 	matchPattern := `error:1` // nolint: goconst
 	ignorePattern := `ignore` // nolint: goconst
 
-	watcher, err := NewWatcher(dbName, filePath, matchPattern, ignorePattern, false)
+	watcher, err := NewWatcher(dbName, filePath, matchPattern, ignorePattern)
 	assert.NoError(t, err)
 	defer watcher.Close()
 
@@ -112,7 +112,7 @@ error:1`
 	matchPattern := `error:1`
 	ignorePattern := `ignore`
 
-	watcher, err := NewWatcher(dbName, filePath, matchPattern, ignorePattern, false)
+	watcher, err := NewWatcher(dbName, filePath, matchPattern, ignorePattern)
 	if err != nil {
 		b.Fatal(err)
 	}
@@ -132,7 +132,7 @@ func BenchmarkLoadAndSaveState(b *testing.B) {
 	matchPattern := "error:1"
 	ignorePattern := "ignore"
 
-	watcher, err := NewWatcher(dbName, filePath, matchPattern, ignorePattern, false)
+	watcher, err := NewWatcher(dbName, filePath, matchPattern, ignorePattern)
 	if err != nil {
 		b.Fatal(err)
 	}
@@ -141,7 +141,7 @@ func BenchmarkLoadAndSaveState(b *testing.B) {
 	watcher.lastLineNum = 10
 
 	for i := 0; i < b.N; i++ {
-		_, err := NewWatcher(dbName, filePath, matchPattern, ignorePattern, false)
+		_, err := NewWatcher(dbName, filePath, matchPattern, ignorePattern)
 		if err != nil {
 			b.Fatal(err)
 		}
@@ -162,7 +162,7 @@ line2`
 	matchPattern := `error:1`
 	ignorePattern := `ignore`
 
-	watcher, err := NewWatcher(dbName, filePath, matchPattern, ignorePattern, false)
+	watcher, err := NewWatcher(dbName, filePath, matchPattern, ignorePattern)
 	if err != nil {
 		b.Fatal(err)
 	}