Increase stopBalancer timeout; Improve logging

Signed-off-by: Arnob kumar saha <[email protected]>
stashed · Sep 22, 2023 · 711a134 · 711a134
1 parent 4eaddcf
commit 711a134
Showing 1 changed file with 80 additions and 9 deletions.
diff --git a/pkg/backup.go b/pkg/backup.go
@@ -17,6 +17,7 @@ limitations under the License.
 package pkg
 
 import (
+	"bytes"
 	"context"
 	"encoding/json"
 	"fmt"
@@ -607,6 +608,7 @@ func getPrimaryNSecondaryMember(mongoDSN string) (primary, secondary string, err
 		}
 
 		if secHost != primary {
+			klog.Infof("Primary %s & Secondary %s found for mongoDSN %s \n", primary, secHost, mongoDSN)
 			return primary, secHost, nil
 		}
 	}
@@ -623,10 +625,22 @@ func disabelBalancer(mongosHost string) error {
 		"config",
 		"--host", mongosHost,
 		"--quiet",
-		"--eval", "JSON.stringify(sh.stopBalancer())",
+		"--eval", "JSON.stringify(sh.stopBalancer(600000,1000))",
 	}, mongoCreds...)
 	// disable balancer
-	if err := sh.Command(MongoCMD, args...).Command("/usr/bin/tail", "-1").UnmarshalJSON(&v); err != nil {
+	cmd := sh.Command(MongoCMD, args...)
+	bufOut := bytes.NewBuffer(nil)
+	bufErr := bytes.NewBuffer(nil)
+	cmd.Stdout = bufOut
+	cmd.Stderr = bufErr
+	if err := cmd.Run(); err != nil {
+		klog.Errorf("Error while stopping balancer : %s ; output : %s \n", err.Error(), bufOut.Bytes())
+		return err
+	}
+
+	err := json.NewDecoder(bufOut).Decode(&v)
+	if err != nil {
+		klog.Errorf("Decoding error while stopping balancer : %s \n", err.Error())
 		return err
 	}
 
@@ -642,8 +656,10 @@ func disabelBalancer(mongosHost string) error {
 		"--eval", "while(sh.isBalancerRunning()){ print('waiting for balancer to stop...'); sleep(1000);}",
 	}, mongoCreds...)
 	if err := sh.Command(MongoCMD, args...).Command("/usr/bin/tail", "-1").Run(); err != nil {
+		klog.Errorf("Error while waiting for the balancer to stop : %s \n", err.Error())
 		return err
 	}
+	klog.Info("Balancer successfully Disabled.")
 	return nil
 }
 
@@ -659,14 +675,32 @@ func enableBalancer(mongosHost string) error {
 		"--quiet",
 		"--eval", "JSON.stringify(sh.setBalancerState(true))",
 	}, mongoCreds...)
-	if err := sh.Command(MongoCMD, args...).Command("/usr/bin/tail", "-1").UnmarshalJSON(&v); err != nil {
+	cmd := sh.Command(MongoCMD, args...)
+	bufOut := bytes.NewBuffer(nil)
+	bufErr := bytes.NewBuffer(nil)
+	cmd.Stdout = bufOut
+	cmd.Stderr = bufErr
+	for i := 0; i < 10; i++ {
+		err := cmd.Run()
+		if err != nil {
+			klog.Errorf("Try #%d : Error on setBalancerState command : %s, output : %s .\n", i, err.Error(), bufOut.Bytes())
+			time.Sleep(time.Second)
+		} else {
+			break
+		}
+	}
+
+	err := json.NewDecoder(bufOut).Decode(&v)
+	if err != nil {
+		klog.Errorf("Decoding error while enabling balancer : %+v , bufOut : %s \n", err.Error(), bufOut.Bytes())
 		return err
 	}
 
 	if val, ok := v["ok"].(float64); !ok || int(val) != 1 {
-		return fmt.Errorf("unable to disable balancer. got response: %v", v)
+		return fmt.Errorf("unable to enable balancer. got response: %v", v)
 	}
 
+	klog.Info("Balancer successfully re-enabled.")
 	return nil
 }
 
@@ -685,7 +719,20 @@ func lockConfigServer(configSVRDSN, secondaryHost string) error {
 		"--quiet",
 		"--eval", "db.BackupControl.findAndModify({query: { _id: 'BackupControlDocument' }, update: { $inc: { counter : 1 } }, new: true, upsert: true, writeConcern: { w: 'majority', wtimeout: 15000 }});",
 	}, mongoCreds...)
-	if err := sh.Command(MongoCMD, args...).Command("/usr/bin/tail", "-1").UnmarshalJSON(&v); err != nil {
+
+	cmd := sh.Command(MongoCMD, args...)
+	bufOut := bytes.NewBuffer(nil)
+	bufErr := bytes.NewBuffer(nil)
+	cmd.Stdout = bufOut
+	cmd.Stderr = bufErr
+	if err := cmd.Run(); err != nil {
+		klog.Errorf("Error while running findAndModify to lock configServer : %s ; output : %s \n", err.Error(), bufOut.Bytes())
+		return err
+	}
+
+	err := json.NewDecoder(bufOut).Decode(&v)
+	if err != nil {
+		klog.Errorf("Decoding error while running findAndModify to lock configServer : %s \n", err.Error())
 		return err
 	}
 	val, ok := v["counter"].(float64)
@@ -739,14 +786,27 @@ func lockSecondaryMember(mongohost string) error {
 		"--quiet",
 		"--eval", "JSON.stringify(db.fsyncLock())",
 	}, mongoCreds...)
-	if err := sh.Command(MongoCMD, args...).Command("/usr/bin/tail", "-1").UnmarshalJSON(&v); err != nil {
+
+	cmd := sh.Command(MongoCMD, args...)
+	bufOut := bytes.NewBuffer(nil)
+	bufErr := bytes.NewBuffer(nil)
+	cmd.Stdout = bufOut
+	cmd.Stderr = bufErr
+	if err := cmd.Run(); err != nil {
+		klog.Errorf("Error while running fsyncLock on secondary : %s ; output : %s \n", err.Error(), bufOut.Bytes())
+		return err
+	}
+
+	err := json.NewDecoder(bufOut).Decode(&v)
+	if err != nil {
+		klog.Errorf("Decoding error while running fsyncLock on secondary : %s \n", err.Error())
 		return err
 	}
 
 	if val, ok := v["ok"].(float64); !ok || int(val) != 1 {
 		return fmt.Errorf("unable to lock the secondary host. got response: %v", v)
 	}
-
+	klog.Infof("secondary %s locked.", mongohost)
 	return nil
 }
 
@@ -765,13 +825,24 @@ func unlockSecondaryMember(mongohost string) error {
 		"--quiet",
 		"--eval", "JSON.stringify(db.fsyncUnlock())",
 	}, mongoCreds...)
-	if err := sh.Command(MongoCMD, args...).Command("/usr/bin/tail", "-1").UnmarshalJSON(&v); err != nil {
+	cmd := sh.Command(MongoCMD, args...)
+	bufOut := bytes.NewBuffer(nil)
+	bufErr := bytes.NewBuffer(nil)
+	cmd.Stdout = bufOut
+	cmd.Stderr = bufErr
+	if err := cmd.Run(); err != nil {
+		klog.Errorf("Error while running fsyncUnlock on secondary : %s ; output : %s \n", err.Error(), bufOut.Bytes())
 		return err
 	}
 
+	err := json.NewDecoder(bufOut).Decode(&v)
+	if err != nil {
+		klog.Errorf("Decoding error while running fsyncUnlock on secondary : %s \n", err.Error())
+		return err
+	}
 	if val, ok := v["ok"].(float64); !ok || int(val) != 1 {
 		return fmt.Errorf("unable to lock the secondary host. got response: %v", v)
 	}
-
+	klog.Infof("secondary %s unlocked.", mongohost)
 	return nil
 }