From 7d80274f1826d7af590d33f447eeb31fc1e43881 Mon Sep 17 00:00:00 2001 From: Iain Findlay Date: Thu, 5 Sep 2024 17:26:28 +0200 Subject: [PATCH] we should block in shutdown after shutdown was called I O U a proper message --- cmd/serve.go | 10 ++++------ integrationtest/termination_recovery_test.go | 15 +++++++++------ 2 files changed, 13 insertions(+), 12 deletions(-) diff --git a/cmd/serve.go b/cmd/serve.go index 74b005698..70c40b5d8 100644 --- a/cmd/serve.go +++ b/cmd/serve.go @@ -253,19 +253,17 @@ func startServer(registry pakBroker.BrokerRegistry, db *sql.DB, brokerapi http.H switch signalReceived { case syscall.SIGTERM: - logger.Info("received SIGTERM, server is shutting down gracefully allowing for in flight work to finish") - shutdownCtx, shutdownRelease := context.WithTimeout(context.Background(), shutdownTimeout) + if err := httpServer.Shutdown(shutdownCtx); err != nil { + logger.Fatal("shutdown error: %v", err) + } + logger.Info("received SIGTERM, server is shutting down gracefully allowing for in flight work to finish") defer shutdownRelease() for store.LockFilesExist() { logger.Info("draining csb instance") time.Sleep(time.Second * 1) } logger.Info("draining complete") - if err := httpServer.Shutdown(shutdownCtx); err != nil { - logger.Fatal("shutdown error: %v", err) - } - logger.Info("shutdown complete") case syscall.SIGKILL: logger.Info("received SIGKILL, server is shutting down immediately. In flight operations will not finish and their state is potentially lost.") diff --git a/integrationtest/termination_recovery_test.go b/integrationtest/termination_recovery_test.go index c54c082dd..6eb822170 100644 --- a/integrationtest/termination_recovery_test.go +++ b/integrationtest/termination_recovery_test.go @@ -51,17 +51,20 @@ var _ = Describe("Recovery From Broker Termination", func() { Expect(response.StatusCode).To(Equal(http.StatusAccepted)) Eventually(stdout, time.Second*5).Should(gbytes.Say(`tofu","apply","-auto-approve"`)) By("gracefully stopping the broker") + // Stop seems to be blocking, so run it in a routine so we can check that the broker actually rejects requests until it's fully stopped. + go func() { + Expect(broker.Stop()).To(Succeed()) + }() - Expect(broker.Stop()).To(Succeed()) + By("logging a message") + Eventually(stdout).Should(gbytes.Say("received SIGTERM")) By("ensuring that the broker rejects requests") Expect(broker.Client.LastOperation(instanceGUID, uuid.NewString()).Error).To(HaveOccurred()) - By("logging a message") - Expect(string(stdout.Contents())).To(ContainSubstring("received SIGTERM")) - Expect(string(stdout.Contents())).To(ContainSubstring("draining csb")) - Expect(string(stdout.Contents())).To(ContainSubstring("draining complete")) - Expect(string(stdout.Contents())).ToNot(ContainSubstring("shutdown error")) + Eventually(stdout, time.Minute).Should(Say("draining csb")) + Eventually(stdout, time.Minute).Should(Say("draining complete")) + Eventually(stdout, time.Minute).ShouldNot(Say("shutdown error")) broker = must(testdrive.StartBroker(csb, brokerpak, database, testdrive.WithOutputs(stdout, stderr)))