Add liveness probe

main.go

@@ -4,6 +4,8 @@ import (
 	"context"
 	"log"
 	"net/http"
+	"sync/atomic"
+	"time"
 
 	"github.com/kelseyhightower/envconfig"
 
@@ -25,6 +27,7 @@ func main() {
 		Addr:    conf.AccessControlHost,
 		Timeout: conf.AccessControlTimeout,
 	}
+	probe := &livenessProbe{}
 
 	// Sync badge access from keycloak if configured
 	var kc *keycloak.Keycloak
@@ -49,6 +52,7 @@ func main() {
 			}()
 		}
 
+		probe.Add(&c.LastSync)
 		go c.Run(ctx)
 	}
 
@@ -60,8 +64,32 @@ func main() {
 		if err != nil {
 			log.Fatalf("error while configuring reporting controller: %s", err)
 		}
+		probe.Add(&ctrl.LastSync)
 		go ctrl.Run(ctx)
 	}
 
 	<-ctx.Done() // sleep forever while things run in other goroutines
 }
+
+// This is a very crude probe to kick the process if the loops get stuck for some reason.
+type livenessProbe struct {
+	checks []*atomic.Pointer[time.Time]
+}
+
+func (l *livenessProbe) Add(ptr *atomic.Pointer[time.Time]) { l.checks = append(l.checks, ptr) }
+
+func (l *livenessProbe) HandleHTTP(w http.ResponseWriter, r *http.Request) {
+	var mostRecent time.Time
+	for _, check := range l.checks {
+		ts := check.Load()
+		if ts != nil && ts.After(mostRecent) {
+			mostRecent = *ts
+		}
+	}
+
+	if time.Since(mostRecent) > time.Minute*2 {
+		log.Printf("failing liveness probe!")
+		w.WriteHeader(500)
+	}
+	w.WriteHeader(200)
+}

reporting/controller.go

@@ -6,6 +6,7 @@ import (
 	"fmt"
 	"log"
 	"strings"
+	"sync/atomic"
 	"time"
 
 	"github.com/jackc/pgx"
@@ -32,6 +33,8 @@ CREATE INDEX IF NOT EXISTS idx_swipes_time ON swipes (time);
 `
 
 type Controller struct {
+	LastSync atomic.Pointer[time.Time]
+
 	db                  *pgxpool.Pool
 	client              *client.Client
 	keycloak            *keycloak.Keycloak
@@ -63,6 +66,8 @@ func (c *Controller) Run(ctx context.Context) {
 		if err != nil {
 			log.Printf("error scraping swipe events: %s", err)
 		}
+		now := time.Now()
+		c.LastSync.Store(&now)
 		return err == nil
 	})
 }

sync/controller.go

@@ -7,6 +7,7 @@ import (
 	"log"
 	"net/http"
 	"strings"
+	"sync/atomic"
 	"time"
 
 	"github.com/TheLab-ms/access-controller-controller/client"
@@ -27,6 +28,8 @@ type userStorage interface {
 }
 
 type Controller struct {
+	LastSync atomic.Pointer[time.Time]
+
 	controller accessController
 	storage    userStorage
 	conf       *conf.Env
@@ -88,6 +91,8 @@ func (c *Controller) Run(ctx context.Context) {
 
 	start:
 		changed, err := c.sync(ctx)
+		now := time.Now()
+		c.LastSync.Store(&now)
 		if err != nil {
 			log.Printf("sync error: %s", err)
 		} else {