Gracefully stop services on SIGTERM and SIGINT signals (#2239)

beacon/blockchain/service.go

@@ -217,3 +217,9 @@ func (s *Service[
 
 	return nil
 }
+
+func (s *Service[
+	_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _,
+]) Stop() error {
+	return nil
+}

beacon/validator/service.go

@@ -152,3 +152,7 @@ func (s *Service[
 ) error {
 	return nil
 }
+
+func (s *Service[_, _, _, _, _, _, _, _, _, _, _, _, _, _]) Stop() error {
+	return nil
+}

consensus/cometbft/service/service.go

@@ -167,8 +167,7 @@ func (s *Service[_]) Start(
 	return s.node.Start()
 }
 
-// Close is called in start cmd to gracefully cleanup resources.
-func (s *Service[_]) Close() error {
+func (s *Service[_]) Stop() error {
 	var errs []error
 
 	if s.node != nil && s.node.IsRunning() {

execution/client/client.go

@@ -144,6 +144,10 @@ func (s *EngineClient[
 	}
 }
 
+func (s *EngineClient[_, _]) Stop() error {
+	return nil
+}
+
 func (s *EngineClient[_, _]) IsConnected() bool {
 	s.connectedMu.RLock()
 	defer s.connectedMu.RUnlock()

node-api/server/server.go

@@ -88,6 +88,10 @@ func (s *Server[_]) start(ctx context.Context) {
 	}
 }
 
+func (s *Server[_]) Stop() error {
+	return nil
+}
+
 // Name returns the name of the API server service.
 func (s *Server[_]) Name() string {
 	return "node-api-server"

node-core/node/node.go

@@ -69,11 +69,26 @@ func (n *node) Start(
 
 	// Start all the registered services.
 	if err := n.registry.StartAll(gctx); err != nil {
+		// Make sure the services that were successfully started are stopped
+		// before exiting. We assume that it is safe to call Stop on a
+		// service that was never started so we can call StopAll here
+		//#nosec:G703 // ok to ignore this
+		_ = n.registry.StopAll()
 		return err
 	}
 
 	// Wait for those aforementioned exit signals.
-	return g.Wait()
+	err := g.Wait()
+	if err != nil {
+		return err
+	}
+
+	// Stopp each service allowing them the exit gracefully.
+	if err = n.registry.StopAll(); err != nil {
+		return err
+	}
+
+	return nil
 }
 
 // listenForQuitSignals listens for SIGINT and SIGTERM. When a signal is

node-core/services/registry/registry.go

@@ -31,6 +31,9 @@ import (
 type Basic interface {
 	// Start spawns any goroutines required by the service.
 	Start(ctx context.Context) error
+	// Stop stops the service. It should be safe to call
+	// Stop on a service that has not been started
+	Stop() error
 	// Name returns the name of the service.
 	Name() string
 }
@@ -85,6 +88,23 @@ func (s *Registry) StartAll(ctx context.Context) error {
 	return nil
 }
 
+func (s *Registry) StopAll() error {
+	s.logger.Info("Stopping services", "num", len(s.serviceTypes))
+	for _, typeName := range s.serviceTypes {
+		s.logger.Info("Stopping service", "type", typeName)
+		svc := s.services[typeName]
+		if svc == nil {
+			s.logger.Error("service not found", "type", typeName)
+			continue
+		}
+
+		if err := svc.Stop(); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
 // RegisterService appends a service constructor function to the service
 // registry.
 func (s *Registry) RegisterService(service Basic) error {

node-core/services/version/version.go

@@ -144,6 +144,10 @@ func (rs *ReportingService[_, _]) Start(ctx context.Context) error {
 	return nil
 }
 
+func (rs *ReportingService[_, _]) Stop() error {
+	return nil
+}
+
 func (rs *ReportingService[_, _]) printToConsole(
 	ethClient engineprimitives.ClientVersionV1) {
 	rs.logger.Info(fmt.Sprintf(`

observability/telemetry/service.go

@@ -53,3 +53,7 @@ func (s *Service) Name() string {
 func (s *Service) Start(context.Context) error {
 	return nil
 }
+
+func (s *Service) Stop() error {
+	return nil
+}