From 1b16eda537ed45a7b1a4e70f13044c137e8d1110 Mon Sep 17 00:00:00 2001 From: "daniel.meyer" Date: Tue, 8 Feb 2022 11:56:12 +0100 Subject: [PATCH 1/3] .Update health monitor to silence error-based alerts --- health/monitor.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/health/monitor.go b/health/monitor.go index 61329d1..72bbae8 100644 --- a/health/monitor.go +++ b/health/monitor.go @@ -130,7 +130,10 @@ func (m *Monitor) evaluateChecks(at time.Time) checkEvaluationResult { result.numErrors++ logEvent = m.logger.Error() } - logEvent.Err(err).Msgf("Check - '%s'", name) + logEvent.Err(err). + // don't propagate errors to alerting + Bool("no_alert",true). + Msgf("Check - '%s'", name) } if m.onCheckCallback != nil { From bf87b66b768b6ba00523832989682ab2e24aef80 Mon Sep 17 00:00:00 2001 From: "daniel.meyer" Date: Mon, 14 Feb 2022 09:50:48 +0100 Subject: [PATCH 2/3] .Update silence alert for stopping errors --- .gitignore | 4 ++++ shutdown/shutdownHandler.go | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 1b9f31d..3b64d3a 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,6 @@ coverage.out + +.idea/ + +go-base.iml diff --git a/shutdown/shutdownHandler.go b/shutdown/shutdownHandler.go index 00f3c55..ac6d63c 100644 --- a/shutdown/shutdownHandler.go +++ b/shutdown/shutdownHandler.go @@ -83,7 +83,7 @@ func stop(orderedStopables []Stopable, logger zerolog.Logger) { logger.Debug().Msgf("Stopping %s ...", name) err := stopable.Stop() if err != nil { - logger.Error().Err(err).Msgf("Failed stopping '%s'", name) + logger.Error().Err(err).Bool("no_alert",true).Msgf("Failed stopping '%s'", name) continue } logger.Info().Msgf("%s stopped.", name) From 9b2b5029ac9ca875fcb9769c07a0292b74618066 Mon Sep 17 00:00:00 2001 From: Thomas Obenaus Date: Tue, 8 Mar 2022 22:04:25 +0100 Subject: [PATCH 3/3] fix race condition when registering stopables at the shutdown handler --- shutdown/shutdownHandler.go | 10 +++++++++- shutdown/shutdownHandler_test.go | 4 ++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/shutdown/shutdownHandler.go b/shutdown/shutdownHandler.go index ac6d63c..fb51ee2 100644 --- a/shutdown/shutdownHandler.go +++ b/shutdown/shutdownHandler.go @@ -18,6 +18,8 @@ type Handler struct { wg sync.WaitGroup orderedStopables []Stopable + + mux *sync.Mutex } // InstallHandler installs a handler for syscall.SIGINT, syscall.SIGTERM @@ -29,6 +31,7 @@ func InstallHandler(orderedStopables []Stopable, logger zerolog.Logger) *Handler logger: logger, isShutdownPending: atomic.NewBool(false), orderedStopables: make([]Stopable, 0), + mux: &sync.Mutex{}, } handler.orderedStopables = append(handler.orderedStopables, orderedStopables...) @@ -50,6 +53,8 @@ func (h *Handler) Register(stopable Stopable, front ...bool) { pushFront = front[0] } + h.mux.Lock() + defer h.mux.Unlock() if pushFront { h.orderedStopables = append([]Stopable{stopable}, h.orderedStopables...) } else { @@ -67,6 +72,8 @@ func (h *Handler) shutdownHandler(shutdownChan <-chan os.Signal, logger zerolog. logger.Info().Msgf("Received %v. Shutting down...", s) // Stop all components + h.mux.Lock() + defer h.mux.Unlock() stop(h.orderedStopables, logger) } @@ -78,12 +85,13 @@ func (h *Handler) WaitUntilSignal() { // stop calls Stop() on all Stopable in the list as they are ordered. func stop(orderedStopables []Stopable, logger zerolog.Logger) { + for _, stopable := range orderedStopables { name := stopable.String() logger.Debug().Msgf("Stopping %s ...", name) err := stopable.Stop() if err != nil { - logger.Error().Err(err).Bool("no_alert",true).Msgf("Failed stopping '%s'", name) + logger.Error().Err(err).Bool("no_alert", true).Msgf("Failed stopping '%s'", name) continue } logger.Info().Msgf("%s stopped.", name) diff --git a/shutdown/shutdownHandler_test.go b/shutdown/shutdownHandler_test.go index ff97ecc..8b18b53 100644 --- a/shutdown/shutdownHandler_test.go +++ b/shutdown/shutdownHandler_test.go @@ -3,6 +3,7 @@ package shutdown import ( "fmt" "os" + "sync" "testing" "time" @@ -36,6 +37,7 @@ func Test_ShutdownHandler(t *testing.T) { h := Handler{ orderedStopables: stopables, isShutdownPending: atomic.NewBool(false), + mux: &sync.Mutex{}, } shutDownChan := make(chan os.Signal, 1) @@ -71,6 +73,7 @@ func Test_RegisterFront(t *testing.T) { h := Handler{ orderedStopables: make([]Stopable, 0), isShutdownPending: atomic.NewBool(false), + mux: &sync.Mutex{}, } shutDownChan := make(chan os.Signal, 1) @@ -111,6 +114,7 @@ func Test_RegisterBack(t *testing.T) { h := Handler{ orderedStopables: make([]Stopable, 0), isShutdownPending: atomic.NewBool(false), + mux: &sync.Mutex{}, } shutDownChan := make(chan os.Signal, 1)