From d978cfe0c896760dab65d9999cc06c30181416cd Mon Sep 17 00:00:00 2001 From: Farukh Khan Date: Wed, 9 Aug 2023 06:40:31 +0800 Subject: [PATCH 1/3] NET-507 * Fixed server restart changing the node expiration date to the defaults. * Removed expired nodes removal from zombie cleanup routine. * Added a new expired nodes deletion routine which removes expired nodes every hour. --- logic/nodes.go | 31 +++++++++++++++++++++++++++++++ logic/zombie.go | 4 ++-- main.go | 1 + models/node.go | 4 +++- 4 files changed, 37 insertions(+), 3 deletions(-) diff --git a/logic/nodes.go b/logic/nodes.go index 93ed106dd..84414bc29 100644 --- a/logic/nodes.go +++ b/logic/nodes.go @@ -1,6 +1,7 @@ package logic import ( + "context" "encoding/json" "errors" "fmt" @@ -20,6 +21,7 @@ import ( "github.com/gravitl/netmaker/models" "github.com/gravitl/netmaker/servercfg" "github.com/gravitl/netmaker/validation" + "golang.org/x/exp/slog" ) var ( @@ -444,6 +446,35 @@ func GetAllNodesAPI(nodes []models.Node) []models.ApiNode { return apiNodes[:] } +// DeleteExpiredNodes - goroutine which deletes nodes which are expired +func DeleteExpiredNodes(ctx context.Context, peerUpdate chan *models.Node) { + for { + select { + case <-ctx.Done(): + close(peerUpdate) + return + case <-time.After(time.Hour): + // Delete Expired Nodes Every Hour + allnodes, err := GetAllNodes() + if err != nil { + slog.Error("failed to retrieve all nodes", "error", err.Error()) + return + } + for _, node := range allnodes { + if time.Now().After(node.ExpirationDateTime) { + if err := DeleteNode(&node, true); err != nil { + slog.Error("error deleting expired node", "nodeid", node.ID.String(), "error", err.Error()) + continue + } + node.Action = models.NODE_DELETE + peerUpdate <- &node + slog.Info("deleting expired node", "nodeid", node.ID.String()) + } + } + } + } +} + // == PRO == func updateProNodeACLS(node *models.Node) error { diff --git a/logic/zombie.go b/logic/zombie.go index e51365fc0..c7e858a88 100644 --- a/logic/zombie.go +++ b/logic/zombie.go @@ -37,7 +37,7 @@ func CheckZombies(newnode *models.Node) { //skip self continue } - if node.HostID == newnode.HostID || time.Now().After(node.ExpirationDateTime) { + if node.HostID == newnode.HostID { logger.Log(0, "adding ", node.ID.String(), " to zombie list") newZombie <- node.ID } @@ -97,7 +97,7 @@ func ManageZombies(ctx context.Context, peerUpdate chan *models.Node) { zombies = append(zombies[:i], zombies[i+1:]...) continue } - if time.Since(node.LastCheckIn) > time.Minute*ZOMBIE_DELETE_TIME || time.Now().After(node.ExpirationDateTime) { + if time.Since(node.LastCheckIn) > time.Minute*ZOMBIE_DELETE_TIME { if err := DeleteNode(&node, true); err != nil { logger.Log(1, "error deleting zombie node", zombies[i].String(), err.Error()) continue diff --git a/main.go b/main.go index 0bf365e5a..a0fe2824f 100644 --- a/main.go +++ b/main.go @@ -168,6 +168,7 @@ func runMessageQueue(wg *sync.WaitGroup, ctx context.Context) { go func() { peerUpdate := make(chan *models.Node) go logic.ManageZombies(ctx, peerUpdate) + go logic.DeleteExpiredNodes(ctx, peerUpdate) for nodeUpdate := range peerUpdate { if err := mq.NodeUpdate(nodeUpdate); err != nil { logger.Log(0, "failed to send peer update for deleted node: ", nodeUpdate.ID.String(), err.Error()) diff --git a/models/node.go b/models/node.go index a4104ac39..ee10f5597 100644 --- a/models/node.go +++ b/models/node.go @@ -341,7 +341,9 @@ func (node *Node) SetLastPeerUpdate() { // Node.SetExpirationDateTime - sets node expiry time func (node *Node) SetExpirationDateTime() { - node.ExpirationDateTime = time.Now().Add(TEN_YEARS_IN_SECONDS) + if node.ExpirationDateTime.IsZero() { + node.ExpirationDateTime = time.Now().Add(TEN_YEARS_IN_SECONDS) + } } // Node.SetDefaultName - sets a random name to node From d15db1d16eb94cc0cf657a6a69c9bd89ce244fa8 Mon Sep 17 00:00:00 2001 From: Farukh Khan Date: Wed, 9 Aug 2023 19:42:29 +0800 Subject: [PATCH 2/3] NET-507 suggested changes * Possible fix for zombie nodes upon node deletion from the UI. * Suggested changes implemented for expired nodes deletion go routine. --- controllers/hosts.go | 8 ++++---- logic/nodes.go | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/controllers/hosts.go b/controllers/hosts.go index f712c25d1..d5c5b9bf6 100644 --- a/controllers/hosts.go +++ b/controllers/hosts.go @@ -328,13 +328,13 @@ func deleteHostFromNetwork(w http.ResponseWriter, r *http.Request) { // unset all the relayed nodes logic.SetRelayedNodes(false, node.ID.String(), node.RelayedNodes) } - node.Action = models.NODE_DELETE - node.PendingDelete = true - logger.Log(1, "deleting node", node.ID.String(), "from host", currHost.Name) - if err := logic.DeleteNode(node, forceDelete); err != nil { + logger.Log(1, "deleting node", node.ID.String(), "from host", currHost.Name) + if err := logic.DeleteNode(node, false); err != nil { logic.ReturnErrorResponse(w, r, logic.FormatError(fmt.Errorf("failed to delete node"), "internal")) return } + node.Action = models.NODE_DELETE + node.PendingDelete = true // notify node change runUpdates(node, false) go func() { // notify of peer change diff --git a/logic/nodes.go b/logic/nodes.go index 84414bc29..7c85b9982 100644 --- a/logic/nodes.go +++ b/logic/nodes.go @@ -451,7 +451,6 @@ func DeleteExpiredNodes(ctx context.Context, peerUpdate chan *models.Node) { for { select { case <-ctx.Done(): - close(peerUpdate) return case <-time.After(time.Hour): // Delete Expired Nodes Every Hour @@ -462,11 +461,12 @@ func DeleteExpiredNodes(ctx context.Context, peerUpdate chan *models.Node) { } for _, node := range allnodes { if time.Now().After(node.ExpirationDateTime) { - if err := DeleteNode(&node, true); err != nil { + if err := DeleteNode(&node, false); err != nil { slog.Error("error deleting expired node", "nodeid", node.ID.String(), "error", err.Error()) continue } node.Action = models.NODE_DELETE + node.PendingDelete = true peerUpdate <- &node slog.Info("deleting expired node", "nodeid", node.ID.String()) } From 448f07d6957c7a6253560efd41b9594279dbd7e5 Mon Sep 17 00:00:00 2001 From: Farukh Khan Date: Wed, 9 Aug 2023 19:45:32 +0800 Subject: [PATCH 3/3] NET-507 typo fix * typo fix for DeleteNode purge parameter --- controllers/hosts.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/controllers/hosts.go b/controllers/hosts.go index d5c5b9bf6..1cff4aea8 100644 --- a/controllers/hosts.go +++ b/controllers/hosts.go @@ -329,7 +329,7 @@ func deleteHostFromNetwork(w http.ResponseWriter, r *http.Request) { logic.SetRelayedNodes(false, node.ID.String(), node.RelayedNodes) } logger.Log(1, "deleting node", node.ID.String(), "from host", currHost.Name) - if err := logic.DeleteNode(node, false); err != nil { + if err := logic.DeleteNode(node, forceDelete); err != nil { logic.ReturnErrorResponse(w, r, logic.FormatError(fmt.Errorf("failed to delete node"), "internal")) return }