diff --git a/main.go b/main.go index 84b2ecc..d081479 100644 --- a/main.go +++ b/main.go @@ -4,6 +4,8 @@ import ( "context" "log" "net/http" + "sync/atomic" + "time" "github.com/kelseyhightower/envconfig" @@ -25,6 +27,7 @@ func main() { Addr: conf.AccessControlHost, Timeout: conf.AccessControlTimeout, } + probe := &livenessProbe{} // Sync badge access from keycloak if configured var kc *keycloak.Keycloak @@ -49,6 +52,7 @@ func main() { }() } + probe.Add(&c.LastSync) go c.Run(ctx) } @@ -60,8 +64,32 @@ func main() { if err != nil { log.Fatalf("error while configuring reporting controller: %s", err) } + probe.Add(&ctrl.LastSync) go ctrl.Run(ctx) } <-ctx.Done() // sleep forever while things run in other goroutines } + +// This is a very crude probe to kick the process if the loops get stuck for some reason. +type livenessProbe struct { + checks []*atomic.Pointer[time.Time] +} + +func (l *livenessProbe) Add(ptr *atomic.Pointer[time.Time]) { l.checks = append(l.checks, ptr) } + +func (l *livenessProbe) HandleHTTP(w http.ResponseWriter, r *http.Request) { + var mostRecent time.Time + for _, check := range l.checks { + ts := check.Load() + if ts != nil && ts.After(mostRecent) { + mostRecent = *ts + } + } + + if time.Since(mostRecent) > time.Minute*2 { + log.Printf("failing liveness probe!") + w.WriteHeader(500) + } + w.WriteHeader(200) +} diff --git a/reporting/controller.go b/reporting/controller.go index 6f55936..f1d44ec 100644 --- a/reporting/controller.go +++ b/reporting/controller.go @@ -6,6 +6,7 @@ import ( "fmt" "log" "strings" + "sync/atomic" "time" "github.com/jackc/pgx" @@ -32,6 +33,8 @@ CREATE INDEX IF NOT EXISTS idx_swipes_time ON swipes (time); ` type Controller struct { + LastSync atomic.Pointer[time.Time] + db *pgxpool.Pool client *client.Client keycloak *keycloak.Keycloak @@ -63,6 +66,8 @@ func (c *Controller) Run(ctx context.Context) { if err != nil { log.Printf("error scraping swipe events: %s", err) } + now := time.Now() + c.LastSync.Store(&now) return err == nil }) } diff --git a/sync/controller.go b/sync/controller.go index 0a12e52..67173b6 100644 --- a/sync/controller.go +++ b/sync/controller.go @@ -7,6 +7,7 @@ import ( "log" "net/http" "strings" + "sync/atomic" "time" "github.com/TheLab-ms/access-controller-controller/client" @@ -27,6 +28,8 @@ type userStorage interface { } type Controller struct { + LastSync atomic.Pointer[time.Time] + controller accessController storage userStorage conf *conf.Env @@ -88,6 +91,8 @@ func (c *Controller) Run(ctx context.Context) { start: changed, err := c.sync(ctx) + now := time.Now() + c.LastSync.Store(&now) if err != nil { log.Printf("sync error: %s", err) } else {