diff --git a/pkg/backup.go b/pkg/backup.go index dc1d2b1ca..454eabb67 100644 --- a/pkg/backup.go +++ b/pkg/backup.go @@ -17,6 +17,7 @@ limitations under the License. package pkg import ( + "bytes" "context" "encoding/json" "fmt" @@ -607,6 +608,7 @@ func getPrimaryNSecondaryMember(mongoDSN string) (primary, secondary string, err } if secHost != primary { + klog.Infof("Primary %s & Secondary %s found for mongoDSN %s \n", primary, secHost, mongoDSN) return primary, secHost, nil } } @@ -623,10 +625,22 @@ func disabelBalancer(mongosHost string) error { "config", "--host", mongosHost, "--quiet", - "--eval", "JSON.stringify(sh.stopBalancer())", + "--eval", "JSON.stringify(sh.stopBalancer(600000,1000))", }, mongoCreds...) // disable balancer - if err := sh.Command(MongoCMD, args...).Command("/usr/bin/tail", "-1").UnmarshalJSON(&v); err != nil { + cmd := sh.Command(MongoCMD, args...) + bufOut := bytes.NewBuffer(nil) + bufErr := bytes.NewBuffer(nil) + cmd.Stdout = bufOut + cmd.Stderr = bufErr + if err := cmd.Run(); err != nil { + klog.Errorf("Error while stopping balancer : %s ; output : %s \n", err.Error(), bufOut.Bytes()) + return err + } + + err := json.NewDecoder(bufOut).Decode(&v) + if err != nil { + klog.Errorf("Decoding error while stopping balancer : %s \n", err.Error()) return err } @@ -642,8 +656,10 @@ func disabelBalancer(mongosHost string) error { "--eval", "while(sh.isBalancerRunning()){ print('waiting for balancer to stop...'); sleep(1000);}", }, mongoCreds...) if err := sh.Command(MongoCMD, args...).Command("/usr/bin/tail", "-1").Run(); err != nil { + klog.Errorf("Error while waiting for the balancer to stop : %s \n", err.Error()) return err } + klog.Info("Balancer successfully Disabled.") return nil } @@ -659,14 +675,32 @@ func enableBalancer(mongosHost string) error { "--quiet", "--eval", "JSON.stringify(sh.setBalancerState(true))", }, mongoCreds...) - if err := sh.Command(MongoCMD, args...).Command("/usr/bin/tail", "-1").UnmarshalJSON(&v); err != nil { + cmd := sh.Command(MongoCMD, args...) + bufOut := bytes.NewBuffer(nil) + bufErr := bytes.NewBuffer(nil) + cmd.Stdout = bufOut + cmd.Stderr = bufErr + for i := 0; i < 10; i++ { + err := cmd.Run() + if err != nil { + klog.Errorf("Try #%d : Error on setBalancerState command : %s, output : %s .\n", i, err.Error(), bufOut.Bytes()) + time.Sleep(time.Second) + } else { + break + } + } + + err := json.NewDecoder(bufOut).Decode(&v) + if err != nil { + klog.Errorf("Decoding error while enabling balancer : %+v , bufOut : %s \n", err.Error(), bufOut.Bytes()) return err } if val, ok := v["ok"].(float64); !ok || int(val) != 1 { - return fmt.Errorf("unable to disable balancer. got response: %v", v) + return fmt.Errorf("unable to enable balancer. got response: %v", v) } + klog.Info("Balancer successfully re-enabled.") return nil } @@ -685,7 +719,20 @@ func lockConfigServer(configSVRDSN, secondaryHost string) error { "--quiet", "--eval", "db.BackupControl.findAndModify({query: { _id: 'BackupControlDocument' }, update: { $inc: { counter : 1 } }, new: true, upsert: true, writeConcern: { w: 'majority', wtimeout: 15000 }});", }, mongoCreds...) - if err := sh.Command(MongoCMD, args...).Command("/usr/bin/tail", "-1").UnmarshalJSON(&v); err != nil { + + cmd := sh.Command(MongoCMD, args...) + bufOut := bytes.NewBuffer(nil) + bufErr := bytes.NewBuffer(nil) + cmd.Stdout = bufOut + cmd.Stderr = bufErr + if err := cmd.Run(); err != nil { + klog.Errorf("Error while running findAndModify to lock configServer : %s ; output : %s \n", err.Error(), bufOut.Bytes()) + return err + } + + err := json.NewDecoder(bufOut).Decode(&v) + if err != nil { + klog.Errorf("Decoding error while running findAndModify to lock configServer : %s \n", err.Error()) return err } val, ok := v["counter"].(float64) @@ -739,14 +786,27 @@ func lockSecondaryMember(mongohost string) error { "--quiet", "--eval", "JSON.stringify(db.fsyncLock())", }, mongoCreds...) - if err := sh.Command(MongoCMD, args...).Command("/usr/bin/tail", "-1").UnmarshalJSON(&v); err != nil { + + cmd := sh.Command(MongoCMD, args...) + bufOut := bytes.NewBuffer(nil) + bufErr := bytes.NewBuffer(nil) + cmd.Stdout = bufOut + cmd.Stderr = bufErr + if err := cmd.Run(); err != nil { + klog.Errorf("Error while running fsyncLock on secondary : %s ; output : %s \n", err.Error(), bufOut.Bytes()) + return err + } + + err := json.NewDecoder(bufOut).Decode(&v) + if err != nil { + klog.Errorf("Decoding error while running fsyncLock on secondary : %s \n", err.Error()) return err } if val, ok := v["ok"].(float64); !ok || int(val) != 1 { return fmt.Errorf("unable to lock the secondary host. got response: %v", v) } - + klog.Infof("secondary %s locked.", mongohost) return nil } @@ -765,13 +825,24 @@ func unlockSecondaryMember(mongohost string) error { "--quiet", "--eval", "JSON.stringify(db.fsyncUnlock())", }, mongoCreds...) - if err := sh.Command(MongoCMD, args...).Command("/usr/bin/tail", "-1").UnmarshalJSON(&v); err != nil { + cmd := sh.Command(MongoCMD, args...) + bufOut := bytes.NewBuffer(nil) + bufErr := bytes.NewBuffer(nil) + cmd.Stdout = bufOut + cmd.Stderr = bufErr + if err := cmd.Run(); err != nil { + klog.Errorf("Error while running fsyncUnlock on secondary : %s ; output : %s \n", err.Error(), bufOut.Bytes()) return err } + err := json.NewDecoder(bufOut).Decode(&v) + if err != nil { + klog.Errorf("Decoding error while running fsyncUnlock on secondary : %s \n", err.Error()) + return err + } if val, ok := v["ok"].(float64); !ok || int(val) != 1 { return fmt.Errorf("unable to lock the secondary host. got response: %v", v) } - + klog.Infof("secondary %s unlocked.", mongohost) return nil }