From 57923aa7553cb65aafa1f43a50108f2432e56b17 Mon Sep 17 00:00:00 2001 From: Yadu Nandan Date: Sat, 7 May 2022 13:23:37 -0700 Subject: [PATCH 01/20] init commit --- lbcluster/lbcluster.go | 2 + lbcluster/lbcluster_dns.go | 3 +- lbcluster/lbcluster_log.go | 2 + lbconfig/config.go | 199 ++++++++++++++++++++++++--------- lbd.go | 216 ++++++++++++++++++------------------ lbhost/lbhost.go | 3 +- tests/file_watch_test.go | 2 + tests/get_state_dns_test.go | 2 +- tests/loadConfig_test.go | 14 +++ 9 files changed, 280 insertions(+), 163 deletions(-) create mode 100644 tests/file_watch_test.go diff --git a/lbcluster/lbcluster.go b/lbcluster/lbcluster.go index 05998e3..ac24655 100644 --- a/lbcluster/lbcluster.go +++ b/lbcluster/lbcluster.go @@ -292,10 +292,12 @@ func (lbc *LBCluster) EvaluateHosts(hostsToCheck map[string]lbhost.LBHost) { for currenthost := range lbc.Host_metric_table { host := hostsToCheck[currenthost] + //todo: parallelize here ips, err := host.Get_working_IPs() if err != nil { ips, err = host.Get_Ips() } + lbc.Host_metric_table[currenthost] = Node{host.Get_load_for_alias(lbc.Cluster_name), ips} lbc.Write_to_log("DEBUG", fmt.Sprintf("node: %s It has a load of %d", currenthost, lbc.Host_metric_table[currenthost].Load)) } diff --git a/lbcluster/lbcluster_dns.go b/lbcluster/lbcluster_dns.go index fbf8c1a..e67382f 100644 --- a/lbcluster/lbcluster_dns.go +++ b/lbcluster/lbcluster_dns.go @@ -58,7 +58,7 @@ func (lbc *LBCluster) updateDNS(keyName, tsigKey, dnsManager string) error { m.RemoveRRset([]dns.RR{rrRemoveA}) m.RemoveRRset([]dns.RR{rrRemoveAAAA}) - for _, ip := range lbc.Current_best_ips { + for _, ip := range lbc.x { var rrInsert dns.RR if ip.To4() != nil { rrInsert, _ = dns.NewRR(lbc.Cluster_name + ". " + ttl + " IN A " + ip.String()) @@ -73,6 +73,7 @@ func (lbc *LBCluster) updateDNS(keyName, tsigKey, dnsManager string) error { c.TsigSecret = map[string]string{keyName: tsigKey} _, _, err := c.Exchange(m, dnsManager+":53") if err != nil { + // todo: consider retries here lbc.Write_to_log("ERROR", fmt.Sprintf("DNS update failed with (%v)", err)) return err } diff --git a/lbcluster/lbcluster_log.go b/lbcluster/lbcluster_log.go index 6ed72c8..ba68c0a 100644 --- a/lbcluster/lbcluster_log.go +++ b/lbcluster/lbcluster_log.go @@ -18,6 +18,8 @@ type Log struct { logMu sync.Mutex } +// todo: refractor logging and consider log based snapshots +// todo: consider start and end times of dns updates //Logger struct for the Logger interface type Logger interface { Info(s string) error diff --git a/lbconfig/config.go b/lbconfig/config.go index a193f66..e8707d5 100644 --- a/lbconfig/config.go +++ b/lbconfig/config.go @@ -10,12 +10,26 @@ import ( "strconv" "strings" "sync" + "time" "gitlab.cern.ch/lb-experts/golbd/lbcluster" ) +type Config interface { + GetMasterHost() string + GetHeartBeatFileName() string + GetHeartBeatDirPath() string + GetDNSManager() string + GetTSIGKeyPrefix() string + GetTSIGInternalKey() string + GetTSIGExternalKey() string + LockHeartBeatMutex() + UnlockHeartBeatMutex() + WatchFileChange(controlChan <-chan bool, waitGroup *sync.WaitGroup) <-chan ConfigFileChangeSignal + Load() (*LBConfig, []lbcluster.LBCluster, error) +} // Config this is the configuration of the lbd -type Config struct { +type LBConfig struct { Master string HeartbeatFile string HeartbeatPath string @@ -25,69 +39,107 @@ type Config struct { TsigExternalKey string SnmpPassword string DNSManager string - ConfigFile string + configFilePath string + lbLog *lbcluster.Log Clusters map[string][]string Parameters map[string]lbcluster.Params } -// readLines reads a whole file into memory and returns a slice of lines. -func readLines(path string) (lines []string, err error) { - f, err := os.Open(path) - if err != nil { - return nil, err - } - defer f.Close() +type ConfigFileChangeSignal struct { + readSignal bool + readError error +} - sc := bufio.NewScanner(f) - for sc.Scan() { - lines = append(lines, sc.Text()) +func (fs ConfigFileChangeSignal) IsErrorPresent() bool { + return fs.readError != nil +} + +// NewLoadBalancerConfig - instantiates a new load balancer config +func NewLoadBalancerConfig(configFilePath string, lbClusterLog *lbcluster.Log) Config { + return &LBConfig{ + configFilePath: configFilePath, + lbLog: lbClusterLog, } - return lines, sc.Err() } -//LoadClusters checks the syntax of the clusters defined in the configuration file -func LoadClusters(config *Config, lg *lbcluster.Log) ([]lbcluster.LBCluster, error) { - var lbc lbcluster.LBCluster - var lbcs []lbcluster.LBCluster +func (c *LBConfig) GetMasterHost() string { + return c.Master +} - for k, v := range config.Clusters { - if len(v) == 0 { - lg.Warning("cluster: " + k + " ignored as it has no members defined in the configuration file " + config.ConfigFile) - continue - } - if par, ok := config.Parameters[k]; ok { - lbc = lbcluster.LBCluster{Cluster_name: k, Loadbalancing_username: "loadbalancing", - Loadbalancing_password: config.SnmpPassword, Parameters: par, - Current_best_ips: []net.IP{}, - Previous_best_ips_dns: []net.IP{}, - Slog: lg} - hm := make(map[string]lbcluster.Node) - for _, h := range v { - hm[h] = lbcluster.Node{Load: 100000, IPs: []net.IP{}} - } - lbc.Host_metric_table = hm - lbcs = append(lbcs, lbc) - lbc.Write_to_log("INFO", "(re-)loaded cluster ") +func (c *LBConfig) GetHeartBeatFileName() string { + return c.HeartbeatFile +} - } else { - lg.Warning("cluster: " + k + " missing parameters for cluster; ignoring the cluster, please check the configuration file " + config.ConfigFile) - } - } +func (c *LBConfig) GetHeartBeatDirPath() string { + return c.HeartbeatPath +} - return lbcs, nil +func (c *LBConfig) GetDNSManager() string { + return c.DNSManager +} + +func (c *LBConfig) GetTSIGKeyPrefix() string{ + return c.TsigKeyPrefix +} + +func (c *LBConfig) GetTSIGInternalKey() string { + return c.TsigInternalKey +} +func (c *LBConfig) GetTSIGExternalKey() string { + return c.TsigExternalKey } -//LoadConfig reads a configuration file and returns a struct with the config -func LoadConfig(configFile string, lg *lbcluster.Log) (*Config, []lbcluster.LBCluster, error) { +func (c *LBConfig) LockHeartBeatMutex() { + c.HeartbeatMu.Lock() +} + +func (c *LBConfig) UnlockHeartBeatMutex(){ + c.HeartbeatMu.Unlock() +} + + +func (c *LBConfig) WatchFileChange(controlChan <-chan bool, waitGroup *sync.WaitGroup) <-chan ConfigFileChangeSignal { + fileWatcherChan := make(chan ConfigFileChangeSignal) + waitGroup.Add(1) + go func() { + defer close(fileWatcherChan) + defer waitGroup.Done() + initialStat, err := os.Stat(c.configFilePath) + if err != nil { + fileWatcherChan <- ConfigFileChangeSignal{readError: err} + } + secondTicker := time.NewTicker(1*time.Second) + for { + select { + case <- secondTicker.C: + stat, err := os.Stat(c.configFilePath) + if err != nil { + fileWatcherChan <- ConfigFileChangeSignal{readError: err} + continue + } + if stat.Size() != initialStat.Size() || stat.ModTime() != initialStat.ModTime() { + fileWatcherChan <- ConfigFileChangeSignal{readSignal: true} + initialStat = stat + } + case <-controlChan: + return + } + } + }() + return fileWatcherChan +} + +//Load reads a configuration file and returns a struct with the config +func (c *LBConfig) Load() (*LBConfig, []lbcluster.LBCluster, error) { var ( - config Config + config LBConfig p lbcluster.Params mc = make(map[string][]string) mp = make(map[string]lbcluster.Params) ) - lines, err := readLines(configFile) + lines, err := readLines(c.configFilePath) if err != nil { return nil, nil, err } @@ -139,29 +191,76 @@ func LoadConfig(configFile string, lg *lbcluster.Log) (*Config, []lbcluster.LBCl break } else if err != nil { //log.Fatal(err) - lg.Warning(fmt.Sprintf("%v", err)) + c.lbLog.Warning(fmt.Sprintf("%v", err)) os.Exit(1) } mp[words[1]] = p } else if words[0] == "clusters" { mc[words[1]] = words[3:] - lg.Debug(words[1]) - lg.Debug(fmt.Sprintf("%v", words[3:])) + c.lbLog.Debug(words[1]) + c.lbLog.Debug(fmt.Sprintf("%v", words[3:])) } } } config.Parameters = mp config.Clusters = mc - config.ConfigFile = configFile - lbclusters, err := LoadClusters(&config, lg) + lbclusters, err := c.loadClusters() if err != nil { fmt.Println("Error getting the clusters") return nil, nil, err } - lg.Info("Clusters loaded") + c.lbLog.Info("Clusters loaded") return &config, lbclusters, nil } + +// readLines reads a whole file into memory and returns a slice of lines. +func readLines(path string) (lines []string, err error) { + f, err := os.Open(path) + if err != nil { + return nil, err + } + defer f.Close() + + sc := bufio.NewScanner(f) + for sc.Scan() { + lines = append(lines, sc.Text()) + } + return lines, sc.Err() +} + +//loadClusters checks the syntax of the clusters defined in the configuration file +func (c *LBConfig) loadClusters() ([]lbcluster.LBCluster, error) { + var lbc lbcluster.LBCluster + var lbcs []lbcluster.LBCluster + + for k, v := range c.Clusters { + if len(v) == 0 { + c.lbLog.Warning("cluster: " + k + " ignored as it has no members defined in the configuration file " + c.configFilePath) + continue + } + if par, ok := c.Parameters[k]; ok { + lbc = lbcluster.LBCluster{Cluster_name: k, Loadbalancing_username: "loadbalancing", + Loadbalancing_password: c.SnmpPassword, Parameters: par, + Current_best_ips: []net.IP{}, + Previous_best_ips_dns: []net.IP{}, + Slog: c.lbLog} + hm := make(map[string]lbcluster.Node) + for _, h := range v { + hm[h] = lbcluster.Node{Load: 100000, IPs: []net.IP{}} + } + lbc.Host_metric_table = hm + lbcs = append(lbcs, lbc) + lbc.Write_to_log("INFO", "(re-)loaded cluster ") + + } else { + c.lbLog.Warning("cluster: " + k + " missing parameters for cluster; ignoring the cluster, please check the configuration file " + c.configFilePath) + } + } + + return lbcs, nil + +} \ No newline at end of file diff --git a/lbd.go b/lbd.go index bf3c810..c31eb7f 100644 --- a/lbd.go +++ b/lbd.go @@ -7,16 +7,16 @@ import ( "log/syslog" "math/rand" "os" - "os/signal" "regexp" "strconv" "strings" - "syscall" + "sync" "time" "gitlab.cern.ch/lb-experts/golbd/lbcluster" - "gitlab.cern.ch/lb-experts/golbd/lbconfig" "gitlab.cern.ch/lb-experts/golbd/lbhost" + + "lb-experts/golbd/lbconfig" ) var ( @@ -32,24 +32,33 @@ var ( startFlag = flag.Bool("start", false, "start lbd") stopFlag = flag.Bool("stop", false, "stop lbd") updateFlag = flag.Bool("update", false, "update lbd config") - configFileFlag = flag.String("config", "./load-balancing.conf", "specify configuration file path") + configFileFlag = flag.String("config", "./load-balancing.conf", "specify configuration file path") logFileFlag = flag.String("log", "./lbd.log", "specify log file path") stdoutFlag = flag.Bool("stdout", false, "send log to stdtout") ) -const itCSgroupDNSserver string = "cfmgr.cern.ch" +const ( + itCSgroupDNSserver string = "cfmgr.cern.ch" + DefaultSleepDuration = 10 + DefaultLbdTag ="lbd" + DefaultConnectionTimeout =10 * time.Second + DefaultReadTimeout =20 * time.Second +) + +type ConfigFileChangeSignal struct { + readSignal bool + readError error +} -func shouldUpdateDNS(config *lbconfig.Config, hostname string, lg *lbcluster.Log) bool { - if hostname == config.Master { +func shouldUpdateDNS(config lbconfig.Config, hostname string, lg *lbcluster.Log) bool { + if strings.EqualFold( hostname, config.GetMasterHost()) { return true } masterHeartbeat := "I am sick" - connectTimeout := (10 * time.Second) - readWriteTimeout := (20 * time.Second) - httpClient := lbcluster.NewTimeoutClient(connectTimeout, readWriteTimeout) - response, err := httpClient.Get("http://" + config.Master + "/load-balancing/" + config.HeartbeatFile) + httpClient := lbcluster.NewTimeoutClient(DefaultConnectionTimeout, DefaultReadTimeout) + response, err := httpClient.Get("http://" + config.GetMasterHost() + "/load-balancing/" + config.HeartbeatFile) if err != nil { - lg.Warning(fmt.Sprintf("problem fetching heartbeat file from the primary master %v: %v", config.Master, err)) + lg.Warning(fmt.Sprintf("problem fetching heartbeat file from the primary master %v: %v", config.GetMasterHost(), err)) return true } defer response.Body.Close() @@ -60,7 +69,7 @@ func shouldUpdateDNS(config *lbconfig.Config, hostname string, lg *lbcluster.Log lg.Debug(fmt.Sprintf("%s", contents)) masterHeartbeat = strings.TrimSpace(string(contents)) lg.Info("primary master heartbeat: " + masterHeartbeat) - r, _ := regexp.Compile(config.Master + ` : (\d+) : I am alive`) + r, _ := regexp.Compile(config.GetMasterHost() + ` : (\d+) : I am alive`) if r.MatchString(masterHeartbeat) { matches := r.FindStringSubmatch(masterHeartbeat) lg.Debug(fmt.Sprintf(matches[1])) @@ -82,103 +91,86 @@ func shouldUpdateDNS(config *lbconfig.Config, hostname string, lg *lbcluster.Log } -func updateHeartbeat(config *lbconfig.Config, hostname string, lg *lbcluster.Log) error { - if hostname != config.Master { +func updateHeartbeat(config lbconfig.Config, hostname string, lg *lbcluster.Log) error { + if hostname != config.GetMasterHost() { return nil } - heartbeatFile := config.HeartbeatPath + "/" + config.HeartbeatFile + "temp" - heartbeatFileReal := config.HeartbeatPath + "/" + config.HeartbeatFile + heartbeatTempFilePath := config.GetHeartBeatDirPath() + "/" + config.GetHeartBeatFileName() + "temp" + heartbeatFileRealFilePath := config.GetHeartBeatDirPath() + "/" + config.GetHeartBeatFileName() + + //todo: read from channel + config.LockHeartBeatMutex() + defer config.UnlockHeartBeatMutex() - config.HeartbeatMu.Lock() - defer config.HeartbeatMu.Unlock() - f, err := os.OpenFile(heartbeatFile, os.O_APPEND|os.O_WRONLY|os.O_CREATE, 0644) + err := updateHeartBeatToFile(heartbeatTempFilePath, hostname, lg) if err != nil { - lg.Error(fmt.Sprintf("can not open %v for writing: %v", heartbeatFile, err)) return err } - now := time.Now() - secs := now.Unix() - _, err = fmt.Fprintf(f, "%v : %v : I am alive\n", hostname, secs) - lg.Info("updating: heartbeat file " + heartbeatFile) - if err != nil { - lg.Info(fmt.Sprintf("can not write to %v: %v", heartbeatFile, err)) - } - f.Close() - if err = os.Rename(heartbeatFile, heartbeatFileReal); err != nil { - lg.Error(fmt.Sprintf("can not rename %v to %v: %v", heartbeatFile, heartbeatFileReal, err)) + // todo: could the file be reused for any other use cases? + if err = os.Rename(heartbeatTempFilePath, heartbeatFileRealFilePath); err != nil { + lg.Error(fmt.Sprintf("can not rename %v to %v: %v", heartbeatTempFilePath, heartbeatFileRealFilePath, err)) return err } return nil } -func installSignalHandler(sighup, sigterm *bool, lg *lbcluster.Log) { - c := make(chan os.Signal, 1) - signal.Notify(c, syscall.SIGTERM, syscall.SIGHUP) - - go func() { - for { - // Block until a signal is received. - sig := <-c - lg.Info(fmt.Sprintf("\nGiven signal: %v\n", sig)) - switch sig { - case syscall.SIGHUP: - *sighup = true - case syscall.SIGTERM: - *sigterm = true - } - } - }() -} - -/* Using this one (instead of fsnotify) -to check also if the file has been moved*/ -func watchFile(filePath string, chanModified chan int) error { - initialStat, err := os.Stat(filePath) +func updateHeartBeatToFile(heartBeatFilePath string, hostname string, lg *lbcluster.Log) error{ + secs := time.Now().Unix() + f, err := os.OpenFile(heartBeatFilePath, os.O_APPEND|os.O_WRONLY|os.O_CREATE, 0644) + defer f.Close() if err != nil { + lg.Error(fmt.Sprintf("can not open %v for writing: %v", heartBeatFilePath, err)) return err } - - for { - stat, err := os.Stat(filePath) - if err == nil { - if stat.Size() != initialStat.Size() || stat.ModTime() != initialStat.ModTime() { - chanModified <- 1 - initialStat = stat - } - } - time.Sleep(1 * time.Second) + _, err = fmt.Fprintf(f, "%v : %v : I am alive\n", hostname, secs) + lg.Info("updating: heartbeat file " + heartBeatFilePath) + if err != nil { + lg.Info(fmt.Sprintf("can not write to %v: %v", heartBeatFilePath, err)) } + return nil } -func sleep(seconds time.Duration, chanModified chan int) error { - for { - chanModified <- 2 - time.Sleep(seconds * time.Second) - } - return nil +func sleep(seconds time.Duration, controlChan <-chan bool, waitGroup *sync.WaitGroup) <-chan bool{ + sleepSignalChan := make(chan bool) + waitGroup.Add(1) + secondsTicker := time.NewTicker(seconds * time.Second) + go func() { + defer waitGroup.Done() + for { + select { + case <- secondsTicker.C: + sleepSignalChan <- true + break + case <- controlChan: + return + } + } + }() + return sleepSignalChan } func main() { + wg:= sync.WaitGroup{} + log, e := syslog.New(syslog.LOG_NOTICE, DefaultLbdTag) + lg := lbcluster.Log{SyslogWriter: log, Stdout: *stdoutFlag, Debugflag: *debugFlag, TofilePath: *logFileFlag} + controlChan := make(chan bool) + defer close(controlChan) + defer wg.Done() + defer lg.Error("The lbd is not supposed to stop") flag.Parse() if *versionFlag { fmt.Printf("This is a proof of concept golbd version: %s-%s \n", Version, Release) os.Exit(0) } rand.Seed(time.Now().UTC().UnixNano()) - log, e := syslog.New(syslog.LOG_NOTICE, "lbd") - if e != nil { fmt.Printf("Error getting a syslog instance %v\nThe service will only write to the logfile %v\n\n", e, *logFileFlag) } - lg := lbcluster.Log{SyslogWriter: log, Stdout: *stdoutFlag, Debugflag: *debugFlag, TofilePath: *logFileFlag} lg.Info("Starting lbd") - - // var sig_hup, sig_term bool - // installSignalHandler(&sig_hup, &sig_term, &lg) - - config, lbclusters, err := lbconfig.LoadConfig(*configFileFlag, &lg) + lbConfig := lbconfig.NewLoadBalancerConfig(*configFileFlag, &lg) + config, lbclusters, err := lbConfig.Load() if err != nil { lg.Warning("loadConfig Error: ") lg.Warning(err.Error()) @@ -186,28 +178,32 @@ func main() { } lg.Info("Clusters loaded") - doneChan := make(chan int) - go watchFile(*configFileFlag, doneChan) - go sleep(10, doneChan) - + fileChangeSignal := lbConfig.WatchFileChange(controlChan, &wg) + intervalTickerSignal := sleep(DefaultSleepDuration, controlChan,&wg) for { - myValue := <-doneChan - if myValue == 1 { - lg.Info("Config Changed") - config, lbclusters, err = lbconfig.LoadConfig(*configFileFlag, &lg) - if err != nil { - lg.Error(fmt.Sprintf("Error getting the clusters (something wrong in %v", configFileFlag)) - } - } else if myValue == 2 { - checkAliases(config, lg, lbclusters) - } else { - lg.Error("Got an unexpected value") + select { + case fileWatcherData := <-fileChangeSignal: + if fileWatcherData.IsErrorPresent(){ + // stop all operations + controlChan <- true + return + } + lg.Info("Config Changed") + config, lbclusters, err = lbConfig.Load() + if err != nil { + lg.Error(fmt.Sprintf("Error getting the clusters (something wrong in %v", configFileFlag)) + } + case <-intervalTickerSignal: + checkAliases(config, lg, lbclusters) + break } } - lg.Error("The lbd is not supposed to stop") - } -func checkAliases(config *lbconfig.Config, lg lbcluster.Log, lbclusters []lbcluster.LBCluster) { +// todo: add some tests +func checkAliases(config lbconfig.Config, lg lbcluster.Log, lbclusters []lbcluster.LBCluster) { + hostCheckChannel := make(chan lbhost.LBHost) + defer close(hostCheckChannel) + hostname, e := os.Hostname() if e == nil { lg.Info("Hostname: " + hostname) @@ -220,27 +216,25 @@ func checkAliases(config *lbconfig.Config, lg lbcluster.Log, lbclusters []lbclus var clustersToUpdate []*lbcluster.LBCluster /* First, let's identify the hosts that have to be checked */ for i := range lbclusters { - pc := &lbclusters[i] - pc.Write_to_log("DEBUG", "DO WE HAVE TO UPDATE?") - if pc.Time_to_refresh() { - pc.Write_to_log("INFO", "Time to refresh the cluster") - pc.Get_list_hosts(hostsToCheck) - clustersToUpdate = append(clustersToUpdate, pc) + currentCluster := &lbclusters[i] + currentCluster.Write_to_log("DEBUG", "DO WE HAVE TO UPDATE?") + if currentCluster.Time_to_refresh() { + currentCluster.Write_to_log("INFO", "Time to refresh the cluster") + currentCluster.Get_list_hosts(hostsToCheck) + clustersToUpdate = append(clustersToUpdate, currentCluster) } } - if len(hostsToCheck) != 0 { - myChannel := make(chan lbhost.LBHost) + if len(hostsToCheck) > 0 { /* Now, let's go through the hosts, issuing the snmp call */ for _, hostValue := range hostsToCheck { go func(myHost lbhost.LBHost) { myHost.Snmp_req() - myChannel <- myHost + hostCheckChannel <- myHost }(hostValue) } - lg.Debug("Let's start gathering the results") - for i := 0; i < len(hostsToCheck); i++ { - myNewHost := <-myChannel - hostsToCheck[myNewHost.Host_name] = myNewHost + lg.Debug("start gathering the results") + for hostChanData := range hostCheckChannel { + hostsToCheck[hostChanData.Host_name] = hostChanData } lg.Debug("All the hosts have been tested") @@ -248,12 +242,14 @@ func checkAliases(config *lbconfig.Config, lg lbcluster.Log, lbclusters []lbclus updateDNS = shouldUpdateDNS(config, hostname, &lg) /* Finally, let's go through the aliases, selecting the best hosts*/ + //todo: try to update clusters in parallel for _, pc := range clustersToUpdate { pc.Write_to_log("DEBUG", "READY TO UPDATE THE CLUSTER") if pc.FindBestHosts(hostsToCheck) { if updateDNS { pc.Write_to_log("DEBUG", "Should update dns is true") - pc.RefreshDNS(config.DNSManager, config.TsigKeyPrefix, config.TsigInternalKey, config.TsigExternalKey) + // todo: try to implement retry mechanism + pc.RefreshDNS(config.GetDNSManager(), config.GetTSIGKeyPrefix(), config.GetTSIGInternalKey(), config.GetTSIGExternalKey()) } else { pc.Write_to_log("DEBUG", "should_update_dns false") } diff --git a/lbhost/lbhost.go b/lbhost/lbhost.go index 30c1312..662546e 100644 --- a/lbhost/lbhost.go +++ b/lbhost/lbhost.go @@ -41,6 +41,7 @@ type LBHost struct { Debugflag bool } +// todo: refractor into smaller functions func (self *LBHost) Snmp_req() { self.find_transports() @@ -128,7 +129,7 @@ func (self *LBHost) Write_to_log(level string, msg string) error { return err } - +// todo: instead of polling try adhoc webhook updates func (self *LBHost) Get_load_for_alias(cluster_name string) int { my_load := -200 diff --git a/tests/file_watch_test.go b/tests/file_watch_test.go new file mode 100644 index 0000000..05c0b06 --- /dev/null +++ b/tests/file_watch_test.go @@ -0,0 +1,2 @@ +package main_test + diff --git a/tests/get_state_dns_test.go b/tests/get_state_dns_test.go index 2b89e78..fe42b38 100644 --- a/tests/get_state_dns_test.go +++ b/tests/get_state_dns_test.go @@ -25,7 +25,7 @@ func TestGetStateDNS(t *testing.T) { "testme007.cern.ch": {[]string{}, nil}, "testme007": {[]string{}, nil}, "kkouros.cern.ch": {[]string{}, nil}, - "aiermis.cern.ch": {[]string{"188.184.104.111", "2001:1458:d00:2d::100:58"}, nil}, + "aiermis.cern.ch": {[]string{}, nil}, } //receiving the output for every alias and storing the results into a map received := make(map[string][]interface{}) diff --git a/tests/loadConfig_test.go b/tests/loadConfig_test.go index f2448ec..d0104d6 100644 --- a/tests/loadConfig_test.go +++ b/tests/loadConfig_test.go @@ -1,8 +1,10 @@ package main import ( + lbconfig2 "lb-experts/golbd/lbconfig" "os" "reflect" + "sync" "testing" "gitlab.cern.ch/lb-experts/golbd/lbcluster" @@ -58,3 +60,15 @@ func TestLoadConfig(t *testing.T) { } } + +func TestWatchConfigFileChanges(t *testing.T) { + lg := lbcluster.Log{Stdout: true, Debugflag: false} + var wg *sync.WaitGroup + var controlChan = make(chan bool) + defer close(controlChan) + config:=lbconfig2.NewLoadBalancerConfig("testloadconfig", &lg) + fileChangeSignal := config.WatchFileChange(controlChan, wg) + for filChangeData := range fileChangeSignal { + + } +} From 705434bfa3d9e15f494ff2203b2132445e429561 Mon Sep 17 00:00:00 2001 From: Yadu Nandan Date: Mon, 9 May 2022 00:16:21 -0700 Subject: [PATCH 02/20] retry module implement init --- lbcluster/lbcluster_dns.go | 1 + lbcluster/retry_module.go | 91 ++++++++++++++++++++++++++++++++++++++ tests/retry_module_test.go | 7 +++ 3 files changed, 99 insertions(+) create mode 100644 lbcluster/retry_module.go create mode 100644 tests/retry_module_test.go diff --git a/lbcluster/lbcluster_dns.go b/lbcluster/lbcluster_dns.go index fbf8c1a..b86afa7 100644 --- a/lbcluster/lbcluster_dns.go +++ b/lbcluster/lbcluster_dns.go @@ -71,6 +71,7 @@ func (lbc *LBCluster) updateDNS(keyName, tsigKey, dnsManager string) error { c := new(dns.Client) m.SetTsig(keyName, dns.HmacMD5, 300, time.Now().Unix()) c.TsigSecret = map[string]string{keyName: tsigKey} + _, _, err := c.Exchange(m, dnsManager+":53") if err != nil { lbc.Write_to_log("ERROR", fmt.Sprintf("DNS update failed with (%v)", err)) diff --git a/lbcluster/retry_module.go b/lbcluster/retry_module.go new file mode 100644 index 0000000..8b978f8 --- /dev/null +++ b/lbcluster/retry_module.go @@ -0,0 +1,91 @@ +package lbcluster + +import "time" + +type Retry struct { + signal chan bool + done chan bool + currentCount int + maxCount int + retryStarted bool + maxDuration time.Duration + retryDuration time.Duration + prevRetryDuration time.Duration +} + +const defaultMaxDuration = 5 * time.Minute + +func NewRetryModule(retryStartDuration time.Duration) *Retry { + retry := &Retry{ + maxCount: -1, + maxDuration: defaultMaxDuration, + retryDuration: retryStartDuration, + prevRetryDuration: retryStartDuration, + } + return retry +} + +func (r *Retry) SetMaxDuration(maxDuration time.Duration) { + if r.retryStarted { + return + } + r.maxDuration = maxDuration + +} + +func (r *Retry) SetMaxCount(maxCount int) { + if r.retryStarted { + return + } + r.maxCount = maxCount +} + +func (r *Retry) Start() <-chan bool { + r.retryStarted = true + signal := make(chan bool) + done := make(chan bool) + end := time.Tick(r.maxDuration) + r.signal = signal + r.done = done + go func() { + defer close(done) + for { + select { + case <-end: + r.done <- true + return + default: + if r.currentCount == r.maxCount { + r.done <- true + return + } + } + } + }() + r.run() + return r.signal +} + +func (r *Retry) run() { + go func() { + defer close(r.signal) + for { + select { + case <-r.done: + return + default: + r.signal <- true + r.currentCount += 1 + time.Sleep(r.retryDuration) + r.computeNextRetryTime() + } + } + }() +} + +// using fibonacci algorithm to compute the next run time +func (r *Retry) computeNextRetryTime() { + nextDuration := r.retryDuration + r.prevRetryDuration + r.prevRetryDuration = r.retryDuration + r.retryDuration = nextDuration +} diff --git a/tests/retry_module_test.go b/tests/retry_module_test.go new file mode 100644 index 0000000..ec2dcfd --- /dev/null +++ b/tests/retry_module_test.go @@ -0,0 +1,7 @@ +package main_test + +import "testing" + +func TestRetryWithMaxDuration(t *testing.T) { + +} From b6459f126de65521bc97261c04e27d02a6538c2c Mon Sep 17 00:00:00 2001 From: Yadu Nandan Date: Mon, 9 May 2022 21:24:40 -0700 Subject: [PATCH 03/20] refractor retry module and added tests --- lbcluster/lbcluster_dns.go | 12 +++-- lbcluster/retry_module.go | 72 ++++++++++++++++++++------- tests/retry_module_test.go | 99 +++++++++++++++++++++++++++++++++++++- 3 files changed, 161 insertions(+), 22 deletions(-) diff --git a/lbcluster/lbcluster_dns.go b/lbcluster/lbcluster_dns.go index b86afa7..d543156 100644 --- a/lbcluster/lbcluster_dns.go +++ b/lbcluster/lbcluster_dns.go @@ -57,7 +57,11 @@ func (lbc *LBCluster) updateDNS(keyName, tsigKey, dnsManager string) error { rrRemoveAAAA, _ := dns.NewRR(lbc.Cluster_name + ". " + ttl + " IN AAAA ::1") m.RemoveRRset([]dns.RR{rrRemoveA}) m.RemoveRRset([]dns.RR{rrRemoveAAAA}) - + retryModule := NewRetryModule(5*time.Second, lbc.Slog) + err := retryModule.SetMaxCount(10) + if err != nil { + return err + } for _, ip := range lbc.Current_best_ips { var rrInsert dns.RR if ip.To4() != nil { @@ -71,8 +75,10 @@ func (lbc *LBCluster) updateDNS(keyName, tsigKey, dnsManager string) error { c := new(dns.Client) m.SetTsig(keyName, dns.HmacMD5, 300, time.Now().Unix()) c.TsigSecret = map[string]string{keyName: tsigKey} - - _, _, err := c.Exchange(m, dnsManager+":53") + err = retryModule.Execute(func() error { + _, _, err := c.Exchange(m, dnsManager+":53") + return err + }) if err != nil { lbc.Write_to_log("ERROR", fmt.Sprintf("DNS update failed with (%v)", err)) return err diff --git a/lbcluster/retry_module.go b/lbcluster/retry_module.go index 8b978f8..5e25134 100644 --- a/lbcluster/retry_module.go +++ b/lbcluster/retry_module.go @@ -1,54 +1,67 @@ package lbcluster -import "time" +import ( + "fmt" + "time" +) type Retry struct { - signal chan bool + signal chan int done chan bool + tick chan bool currentCount int maxCount int retryStarted bool maxDuration time.Duration retryDuration time.Duration prevRetryDuration time.Duration + logger *Log } const defaultMaxDuration = 5 * time.Minute -func NewRetryModule(retryStartDuration time.Duration) *Retry { +func NewRetryModule(retryStartDuration time.Duration, logger *Log) *Retry { retry := &Retry{ maxCount: -1, + currentCount: 0, maxDuration: defaultMaxDuration, retryDuration: retryStartDuration, prevRetryDuration: retryStartDuration, + logger: logger, } return retry } -func (r *Retry) SetMaxDuration(maxDuration time.Duration) { +func (r *Retry) SetMaxDuration(maxDuration time.Duration) error { if r.retryStarted { - return + return nil + } + if maxDuration <= 0 { + return fmt.Errorf("duration has to be greater than 0") } r.maxDuration = maxDuration - + return nil } -func (r *Retry) SetMaxCount(maxCount int) { +func (r *Retry) SetMaxCount(maxCount int) error { if r.retryStarted { - return + return nil + } + if maxCount <= 0 { + return fmt.Errorf("max count has to be greater than 0") } r.maxCount = maxCount + return nil } -func (r *Retry) Start() <-chan bool { +func (r *Retry) start() { r.retryStarted = true - signal := make(chan bool) + signal := make(chan int) done := make(chan bool) end := time.Tick(r.maxDuration) r.signal = signal r.done = done go func() { - defer close(done) for { select { case <-end: @@ -63,24 +76,49 @@ func (r *Retry) Start() <-chan bool { } }() r.run() - return r.signal } func (r *Retry) run() { + start := make(chan bool) go func() { + ticker := time.NewTicker(1 * time.Minute) + defer close(r.done) defer close(r.signal) + defer ticker.Stop() + for { select { case <-r.done: return - default: - r.signal <- true - r.currentCount += 1 - time.Sleep(r.retryDuration) - r.computeNextRetryTime() + case <-ticker.C: + r.nextTick(ticker) + case <-start: + r.nextTick(ticker) } } }() + start <- true +} + +func (r *Retry) nextTick(ticker *time.Ticker) { + r.signal <- r.currentCount + 1 + r.currentCount += 1 + ticker.Reset(r.retryDuration) + r.computeNextRetryTime() +} + +func (r *Retry) Execute(executor func() error) error { + var err error + r.start() + for retryCount := range r.signal { + err = executor() + if err != nil { + r.logger.Debug(fmt.Sprintf("retry count: %v", retryCount)) + } else { + r.done <- true + } + } + return err } // using fibonacci algorithm to compute the next run time diff --git a/tests/retry_module_test.go b/tests/retry_module_test.go index ec2dcfd..78365ac 100644 --- a/tests/retry_module_test.go +++ b/tests/retry_module_test.go @@ -1,7 +1,102 @@ package main_test -import "testing" +import ( + "fmt" + "testing" + "time" -func TestRetryWithMaxDuration(t *testing.T) { + "lb-experts/golbd/lbcluster" +) +func TestRetryWithNoErrorsShouldExitAfterFirstAttempt(t *testing.T) { + lg := &lbcluster.Log{Stdout: true, Debugflag: false} + currentTime := time.Now() + retryModule := lbcluster.NewRetryModule(10*time.Second, lg) + err := retryModule.Execute(func() error { + return nil + }) + if err != nil { + t.Fail() + t.Errorf("error should be nil") + } + if time.Now().Sub(currentTime) > 1*time.Second { + t.Fail() + t.Errorf("should quit after first try") + } +} + +func TestRetryWithErrorShouldQuitAfterMultipleAttempts(t *testing.T) { + lg := &lbcluster.Log{Stdout: true, Debugflag: false} + currentTime := time.Now() + counter := 0 + retryModule := lbcluster.NewRetryModule(1*time.Second, lg) + err := retryModule.Execute(func() error { + if counter == 4 { + return nil + } + counter += 1 + return fmt.Errorf("sample error") + }) + if err != nil { + t.Fail() + t.Errorf("error should be nil") + } + if time.Now().Sub(currentTime) > 12*time.Second { + t.Fail() + t.Errorf("should quit after expected: %v, actual:%v", "11 sec", time.Now().Sub(currentTime)) + } +} + +func TestRetryWithErrorShouldQuitAfterMaxCount(t *testing.T) { + lg := &lbcluster.Log{Stdout: true, Debugflag: false} + currentTime := time.Now() + counter := 0 + retryModule := lbcluster.NewRetryModule(1*time.Second, lg) + err := retryModule.SetMaxCount(3) + if err != nil { + t.Fail() + t.Errorf("error should be nil") + } + err = retryModule.Execute(func() error { + if counter == 4 { + return nil + } + counter += 1 + return fmt.Errorf("sample error") + }) + if err == nil { + t.Fail() + t.Errorf("error should be nil") + } + if time.Now().Sub(currentTime) > 4*time.Second { + t.Fail() + t.Errorf("should quit after expected: %v, actual:%v", "3 sec", time.Now().Sub(currentTime)) + } +} + +func TestRetryWithErrorShouldQuitAfterMaxDuration(t *testing.T) { + lg := &lbcluster.Log{Stdout: true, Debugflag: false} + currentTime := time.Now() + counter := 0 + retryModule := lbcluster.NewRetryModule(1*time.Second, lg) + err := retryModule.SetMaxDuration(4 * time.Second) + if err != nil { + t.Fail() + t.Errorf("error should be nil") + } + err = retryModule.Execute(func() error { + if counter == 4 { + return nil + } + counter += 1 + return fmt.Errorf("sample error") + }) + if err == nil { + t.Fail() + t.Errorf("error should be nil") + } + if time.Now().Sub(currentTime) > 5*time.Second { + t.Fail() + t.Errorf("should quit after expected: %v, actual:%v", "3 sec", time.Now().Sub(currentTime)) + } } From adcbad6608ad6fc4aed707fb2f32a609ddf122cf Mon Sep 17 00:00:00 2001 From: Yadu Nandan Date: Wed, 11 May 2022 01:11:18 -0700 Subject: [PATCH 04/20] refractored lbhost --- lbcluster/lbcluster.go | 78 ++++++++------- lbcluster/lbcluster_dns.go | 14 +-- lbcluster/lbcluster_log.go | 2 +- lbconfig/config.go | 36 ++++--- lbd.go | 84 ++++++++-------- lbhost/lbhost.go | 195 +++++++++++++++++-------------------- tests/aliasload_test.go | 6 +- tests/file_watch_test.go | 2 - 8 files changed, 209 insertions(+), 208 deletions(-) delete mode 100644 tests/file_watch_test.go diff --git a/lbcluster/lbcluster.go b/lbcluster/lbcluster.go index ac24655..d01b94f 100644 --- a/lbcluster/lbcluster.go +++ b/lbcluster/lbcluster.go @@ -4,13 +4,12 @@ import ( "encoding/json" "fmt" "io/ioutil" + "lb-experts/golbd/lbhost" "math/rand" "net" "net/http" "strings" - "gitlab.cern.ch/lb-experts/golbd/lbhost" - "sort" "time" ) @@ -26,9 +25,7 @@ const OID string = ".1.3.6.1.4.1.96.255.1" //LBCluster struct of an lbcluster alias type LBCluster struct { - Cluster_name string - Loadbalancing_username string - Loadbalancing_password string + ClusterConfig Config Host_metric_table map[string]Node Parameters Params Time_of_last_evaluation time.Time @@ -38,6 +35,12 @@ type LBCluster struct { Slog *Log } +type Config struct { + Cluster_name string + Loadbalancing_username string + Loadbalancing_password string +} + //Params of the alias type Params struct { Behaviour string @@ -52,9 +55,9 @@ type Params struct { // Shuffle pseudo-randomizes the order of elements. // n is the number of elements. Shuffle panics if n < 0. // swap swaps the elements with indexes i and j. -func Shuffle(n int, swap func(i, j int)) { +func Shuffle(n int, swap func(i, j int)) error { if n < 0 { - panic("invalid argument to Shuffle") + return fmt.Errorf("invalid argument to Shuffle") } // Fisher-Yates shuffle: https://en.wikipedia.org/wiki/Fisher%E2%80%93Yates_shuffle @@ -72,6 +75,7 @@ func Shuffle(n int, swap func(i, j int)) { j := int(rand.Int31n(int32(i + 1))) swap(i, j) } + return nil } //Node Struct to keep the ips and load of a node for an alias @@ -93,21 +97,15 @@ func (lbc *LBCluster) Time_to_refresh() bool { } //Get_list_hosts Get the hosts for an alias -func (lbc *LBCluster) Get_list_hosts(current_list map[string]lbhost.LBHost) { +func (lbc *LBCluster) Get_list_hosts(current_list map[string]lbhost.Host) { lbc.Write_to_log("DEBUG", "Getting the list of hosts for the alias") for host := range lbc.Host_metric_table { myHost, ok := current_list[host] if ok { - myHost.Cluster_name = myHost.Cluster_name + "," + lbc.Cluster_name + clusterConfig := myHost.GetClusterConfig() + clusterConfig.Cluster_name = clusterConfig.Cluster_name + "," + clusterConfig.Cluster_name } else { - myHost = lbhost.LBHost{ - Cluster_name: lbc.Cluster_name, - Host_name: host, - Loadbalancing_username: lbc.Loadbalancing_username, - Loadbalancing_password: lbc.Loadbalancing_password, - LogFile: lbc.Slog.TofilePath, - Debugflag: lbc.Slog.Debugflag, - } + myHost = lbhost.NewLBHost(lbc.ClusterConfig, lbc.Slog) } current_list[host] = myHost } @@ -133,7 +131,7 @@ func (lbc *LBCluster) concatenateIps(myIps []net.IP) string { } //Find_best_hosts Looks for the best hosts for a cluster -func (lbc *LBCluster) FindBestHosts(hosts_to_check map[string]lbhost.LBHost) bool { +func (lbc *LBCluster) FindBestHosts(hosts_to_check map[string]lbhost.Host) (bool, error) { lbc.EvaluateHosts(hosts_to_check) allMetrics := make(map[string]bool) @@ -144,22 +142,26 @@ func (lbc *LBCluster) FindBestHosts(hosts_to_check map[string]lbhost.LBHost) boo _, ok := allMetrics[lbc.Parameters.Metric] if !ok { lbc.Write_to_log("ERROR", "wrong parameter(metric) in definition of cluster "+lbc.Parameters.Metric) - return false + return false, nil } lbc.Time_of_last_evaluation = time.Now() - if !lbc.ApplyMetric(hosts_to_check) { - return false + shouldApplyMetric, err := lbc.ApplyMetric(hosts_to_check) + if err != nil { + return false, err + } + if !shouldApplyMetric { + return false, nil } nodes := lbc.concatenateIps(lbc.Current_best_ips) if len(lbc.Current_best_ips) == 0 { nodes = "NONE" } lbc.Write_to_log("INFO", "best hosts are: "+nodes) - return true + return true, nil } // ApplyMetric This is the core of the lbcluster: based on the metrics, select the best hosts -func (lbc *LBCluster) ApplyMetric(hosts_to_check map[string]lbhost.LBHost) bool { +func (lbc *LBCluster) ApplyMetric(hosts_to_check map[string]lbhost.Host) (bool, error) { lbc.Write_to_log("INFO", "Got metric = "+lbc.Parameters.Metric) pl := make(NodeList, len(lbc.Host_metric_table)) i := 0 @@ -168,7 +170,10 @@ func (lbc *LBCluster) ApplyMetric(hosts_to_check map[string]lbhost.LBHost) bool i++ } //Let's shuffle the hosts before sorting them, in case some hosts have the same value - Shuffle(len(pl), func(i, j int) { pl[i], pl[j] = pl[j], pl[i] }) + err := Shuffle(len(pl), func(i, j int) { pl[i], pl[j] = pl[j], pl[i] }) + if err != nil { + return false, err + } sort.Sort(pl) lbc.Write_to_log("DEBUG", fmt.Sprintf("%v", pl)) var sorted_host_list []Node @@ -206,7 +211,10 @@ func (lbc *LBCluster) ApplyMetric(hosts_to_check map[string]lbhost.LBHost) bool i++ } //Let's shuffle the hosts - Shuffle(len(pl), func(i, j int) { pl[i], pl[j] = pl[j], pl[i] }) + err := Shuffle(len(pl), func(i, j int) { pl[i], pl[j] = pl[j], pl[i] }) + if err != nil { + return false, err + } for i := 0; i < max; i++ { lbc.Current_best_ips = append(lbc.Current_best_ips, pl[i].IPs...) } @@ -216,7 +224,7 @@ func (lbc *LBCluster) ApplyMetric(hosts_to_check map[string]lbhost.LBHost) bool lbc.Write_to_log("WARNING", "no usable hosts found for cluster! Returning no hosts.") } else if lbc.Parameters.Metric == "cmsfrontier" { lbc.Write_to_log("WARNING", "no usable hosts found for cluster! Skipping the DNS update") - return false + return false, nil } } else { if useful_hosts < max { @@ -228,7 +236,7 @@ func (lbc *LBCluster) ApplyMetric(hosts_to_check map[string]lbhost.LBHost) bool } } - return true + return true, nil } //NewTimeoutClient checks the timeout @@ -288,31 +296,31 @@ func (lbc *LBCluster) checkRogerState(host string) string { } //EvaluateHosts gets the load from the all the nodes -func (lbc *LBCluster) EvaluateHosts(hostsToCheck map[string]lbhost.LBHost) { +func (lbc *LBCluster) EvaluateHosts(hostsToCheck map[string]lbhost.Host) { for currenthost := range lbc.Host_metric_table { host := hostsToCheck[currenthost] //todo: parallelize here - ips, err := host.Get_working_IPs() + ips, err := host.GetWorkingIPs() if err != nil { - ips, err = host.Get_Ips() + ips, err = host.GetIps() } - lbc.Host_metric_table[currenthost] = Node{host.Get_load_for_alias(lbc.Cluster_name), ips} + lbc.Host_metric_table[currenthost] = Node{host.GetLoadForAlias(lbc.ClusterConfig.Cluster_name), ips} lbc.Write_to_log("DEBUG", fmt.Sprintf("node: %s It has a load of %d", currenthost, lbc.Host_metric_table[currenthost].Load)) } } //ReEvaluateHostsForMinimum gets the load from the all the nodes for Minimum metric policy -func (lbc *LBCluster) ReEvaluateHostsForMinimum(hostsToCheck map[string]lbhost.LBHost) { +func (lbc *LBCluster) ReEvaluateHostsForMinimum(hostsToCheck map[string]lbhost.Host) { for currenthost := range lbc.Host_metric_table { host := hostsToCheck[currenthost] - ips, err := host.Get_all_IPs() + ips, err := host.GetAllIPs() if err != nil { - ips, err = host.Get_Ips() + ips, err = host.GetIps() } - lbc.Host_metric_table[currenthost] = Node{host.Get_load_for_alias(lbc.Cluster_name), ips} + lbc.Host_metric_table[currenthost] = Node{host.GetLoadForAlias(lbc.ClusterConfig.Cluster_name), ips} lbc.Write_to_log("DEBUG", fmt.Sprintf("node: %s It has a load of %d", currenthost, lbc.Host_metric_table[currenthost].Load)) } } diff --git a/lbcluster/lbcluster_dns.go b/lbcluster/lbcluster_dns.go index e67382f..9d0ddbf 100644 --- a/lbcluster/lbcluster_dns.go +++ b/lbcluster/lbcluster_dns.go @@ -51,19 +51,19 @@ func (lbc *LBCluster) updateDNS(keyName, tsigKey, dnsManager string) error { } //best_hosts_len := len(lbc.Current_best_hosts) m := new(dns.Msg) - m.SetUpdate(lbc.Cluster_name + ".") + m.SetUpdate(lbc.ClusterConfig.Cluster_name + ".") m.Id = 1234 - rrRemoveA, _ := dns.NewRR(lbc.Cluster_name + ". " + ttl + " IN A 127.0.0.1") - rrRemoveAAAA, _ := dns.NewRR(lbc.Cluster_name + ". " + ttl + " IN AAAA ::1") + rrRemoveA, _ := dns.NewRR(lbc.ClusterConfig.Cluster_name + ". " + ttl + " IN A 127.0.0.1") + rrRemoveAAAA, _ := dns.NewRR(lbc.ClusterConfig.Cluster_name + ". " + ttl + " IN AAAA ::1") m.RemoveRRset([]dns.RR{rrRemoveA}) m.RemoveRRset([]dns.RR{rrRemoveAAAA}) - for _, ip := range lbc.x { + for _, ip := range lbc.Current_best_ips { var rrInsert dns.RR if ip.To4() != nil { - rrInsert, _ = dns.NewRR(lbc.Cluster_name + ". " + ttl + " IN A " + ip.String()) + rrInsert, _ = dns.NewRR(lbc.ClusterConfig.Cluster_name + ". " + ttl + " IN A " + ip.String()) } else if ip.To16() != nil { - rrInsert, _ = dns.NewRR(lbc.Cluster_name + ". " + ttl + " IN AAAA " + ip.String()) + rrInsert, _ = dns.NewRR(lbc.ClusterConfig.Cluster_name + ". " + ttl + " IN AAAA " + ip.String()) } m.Insert([]dns.RR{rrInsert}) } @@ -83,7 +83,7 @@ func (lbc *LBCluster) updateDNS(keyName, tsigKey, dnsManager string) error { } func (lbc *LBCluster) getIpsFromDNS(m *dns.Msg, dnsManager string, dnsType uint16, ips *[]net.IP) error { - m.SetQuestion(lbc.Cluster_name+".", dnsType) + m.SetQuestion(lbc.ClusterConfig.Cluster_name+".", dnsType) in, err := dns.Exchange(m, dnsManager+":53") if err != nil { lbc.Write_to_log("ERROR", fmt.Sprintf("Error getting the ipv4 state of dns: %v", err)) diff --git a/lbcluster/lbcluster_log.go b/lbcluster/lbcluster_log.go index ba68c0a..3df1046 100644 --- a/lbcluster/lbcluster_log.go +++ b/lbcluster/lbcluster_log.go @@ -31,7 +31,7 @@ type Logger interface { //Write_to_log put something in the log file func (lbc *LBCluster) Write_to_log(level string, msg string) error { - myMessage := "cluster: " + lbc.Cluster_name + " " + msg + myMessage := "cluster: " + lbc.ClusterConfig.Cluster_name + " " + msg if level == "INFO" { lbc.Slog.Info(myMessage) diff --git a/lbconfig/config.go b/lbconfig/config.go index e8707d5..e405ce6 100644 --- a/lbconfig/config.go +++ b/lbconfig/config.go @@ -5,14 +5,17 @@ import ( "encoding/json" "fmt" "io" + "lb-experts/golbd/lbcluster" "net" "os" "strconv" "strings" "sync" "time" +) - "gitlab.cern.ch/lb-experts/golbd/lbcluster" +const ( + DefaultLoadBalancerConfig = "loadbalancing" ) type Config interface { @@ -28,6 +31,7 @@ type Config interface { WatchFileChange(controlChan <-chan bool, waitGroup *sync.WaitGroup) <-chan ConfigFileChangeSignal Load() (*LBConfig, []lbcluster.LBCluster, error) } + // Config this is the configuration of the lbd type LBConfig struct { Master string @@ -40,14 +44,14 @@ type LBConfig struct { SnmpPassword string DNSManager string configFilePath string - lbLog *lbcluster.Log + lbLog *lbcluster.Log Clusters map[string][]string Parameters map[string]lbcluster.Params } type ConfigFileChangeSignal struct { readSignal bool - readError error + readError error } func (fs ConfigFileChangeSignal) IsErrorPresent() bool { @@ -58,7 +62,7 @@ func (fs ConfigFileChangeSignal) IsErrorPresent() bool { func NewLoadBalancerConfig(configFilePath string, lbClusterLog *lbcluster.Log) Config { return &LBConfig{ configFilePath: configFilePath, - lbLog: lbClusterLog, + lbLog: lbClusterLog, } } @@ -78,7 +82,7 @@ func (c *LBConfig) GetDNSManager() string { return c.DNSManager } -func (c *LBConfig) GetTSIGKeyPrefix() string{ +func (c *LBConfig) GetTSIGKeyPrefix() string { return c.TsigKeyPrefix } @@ -94,11 +98,10 @@ func (c *LBConfig) LockHeartBeatMutex() { c.HeartbeatMu.Lock() } -func (c *LBConfig) UnlockHeartBeatMutex(){ +func (c *LBConfig) UnlockHeartBeatMutex() { c.HeartbeatMu.Unlock() } - func (c *LBConfig) WatchFileChange(controlChan <-chan bool, waitGroup *sync.WaitGroup) <-chan ConfigFileChangeSignal { fileWatcherChan := make(chan ConfigFileChangeSignal) waitGroup.Add(1) @@ -109,10 +112,10 @@ func (c *LBConfig) WatchFileChange(controlChan <-chan bool, waitGroup *sync.Wait if err != nil { fileWatcherChan <- ConfigFileChangeSignal{readError: err} } - secondTicker := time.NewTicker(1*time.Second) + secondTicker := time.NewTicker(1 * time.Second) for { select { - case <- secondTicker.C: + case <-secondTicker.C: stat, err := os.Stat(c.configFilePath) if err != nil { fileWatcherChan <- ConfigFileChangeSignal{readError: err} @@ -243,11 +246,18 @@ func (c *LBConfig) loadClusters() ([]lbcluster.LBCluster, error) { continue } if par, ok := c.Parameters[k]; ok { - lbc = lbcluster.LBCluster{Cluster_name: k, Loadbalancing_username: "loadbalancing", - Loadbalancing_password: c.SnmpPassword, Parameters: par, + lbcConfig := lbcluster.Config{ + Cluster_name: k, + Loadbalancing_username: DefaultLoadBalancerConfig, + Loadbalancing_password: c.SnmpPassword, + } + lbc = lbcluster.LBCluster{ + ClusterConfig: lbcConfig, + Parameters: par, Current_best_ips: []net.IP{}, Previous_best_ips_dns: []net.IP{}, - Slog: c.lbLog} + Slog: c.lbLog, + } hm := make(map[string]lbcluster.Node) for _, h := range v { hm[h] = lbcluster.Node{Load: 100000, IPs: []net.IP{}} @@ -263,4 +273,4 @@ func (c *LBConfig) loadClusters() ([]lbcluster.LBCluster, error) { return lbcs, nil -} \ No newline at end of file +} diff --git a/lbd.go b/lbd.go index c31eb7f..aa95bd9 100644 --- a/lbd.go +++ b/lbd.go @@ -4,6 +4,9 @@ import ( "flag" "fmt" "io/ioutil" + "lb-experts/golbd/lbcluster" + "lb-experts/golbd/lbhost" + "log" "log/syslog" "math/rand" "os" @@ -13,9 +16,6 @@ import ( "sync" "time" - "gitlab.cern.ch/lb-experts/golbd/lbcluster" - "gitlab.cern.ch/lb-experts/golbd/lbhost" - "lb-experts/golbd/lbconfig" ) @@ -32,31 +32,31 @@ var ( startFlag = flag.Bool("start", false, "start lbd") stopFlag = flag.Bool("stop", false, "stop lbd") updateFlag = flag.Bool("update", false, "update lbd config") - configFileFlag = flag.String("config", "./load-balancing.conf", "specify configuration file path") + configFileFlag = flag.String("config", "./load-balancing.conf", "specify configuration file path") logFileFlag = flag.String("log", "./lbd.log", "specify log file path") stdoutFlag = flag.Bool("stdout", false, "send log to stdtout") ) const ( - itCSgroupDNSserver string = "cfmgr.cern.ch" - DefaultSleepDuration = 10 - DefaultLbdTag ="lbd" - DefaultConnectionTimeout =10 * time.Second - DefaultReadTimeout =20 * time.Second + itCSgroupDNSserver string = "cfmgr.cern.ch" + DefaultSleepDuration = 10 + DefaultLbdTag = "lbd" + DefaultConnectionTimeout = 10 * time.Second + DefaultReadTimeout = 20 * time.Second ) type ConfigFileChangeSignal struct { readSignal bool - readError error + readError error } func shouldUpdateDNS(config lbconfig.Config, hostname string, lg *lbcluster.Log) bool { - if strings.EqualFold( hostname, config.GetMasterHost()) { + if strings.EqualFold(hostname, config.GetMasterHost()) { return true } masterHeartbeat := "I am sick" httpClient := lbcluster.NewTimeoutClient(DefaultConnectionTimeout, DefaultReadTimeout) - response, err := httpClient.Get("http://" + config.GetMasterHost() + "/load-balancing/" + config.HeartbeatFile) + response, err := httpClient.Get("http://" + config.GetMasterHost() + "/load-balancing/" + config.GetHeartBeatFileName()) if err != nil { lg.Warning(fmt.Sprintf("problem fetching heartbeat file from the primary master %v: %v", config.GetMasterHost(), err)) return true @@ -102,7 +102,6 @@ func updateHeartbeat(config lbconfig.Config, hostname string, lg *lbcluster.Log) config.LockHeartBeatMutex() defer config.UnlockHeartBeatMutex() - err := updateHeartBeatToFile(heartbeatTempFilePath, hostname, lg) if err != nil { return err @@ -115,7 +114,7 @@ func updateHeartbeat(config lbconfig.Config, hostname string, lg *lbcluster.Log) return nil } -func updateHeartBeatToFile(heartBeatFilePath string, hostname string, lg *lbcluster.Log) error{ +func updateHeartBeatToFile(heartBeatFilePath string, hostname string, lg *lbcluster.Log) error { secs := time.Now().Unix() f, err := os.OpenFile(heartBeatFilePath, os.O_APPEND|os.O_WRONLY|os.O_CREATE, 0644) defer f.Close() @@ -131,7 +130,7 @@ func updateHeartBeatToFile(heartBeatFilePath string, hostname string, lg *lbclu return nil } -func sleep(seconds time.Duration, controlChan <-chan bool, waitGroup *sync.WaitGroup) <-chan bool{ +func sleep(seconds time.Duration, controlChan <-chan bool, waitGroup *sync.WaitGroup) <-chan bool { sleepSignalChan := make(chan bool) waitGroup.Add(1) secondsTicker := time.NewTicker(seconds * time.Second) @@ -139,10 +138,10 @@ func sleep(seconds time.Duration, controlChan <-chan bool, waitGroup *sync.WaitG defer waitGroup.Done() for { select { - case <- secondsTicker.C: + case <-secondsTicker.C: sleepSignalChan <- true break - case <- controlChan: + case <-controlChan: return } } @@ -151,7 +150,7 @@ func sleep(seconds time.Duration, controlChan <-chan bool, waitGroup *sync.WaitG } func main() { - wg:= sync.WaitGroup{} + wg := sync.WaitGroup{} log, e := syslog.New(syslog.LOG_NOTICE, DefaultLbdTag) lg := lbcluster.Log{SyslogWriter: log, Stdout: *stdoutFlag, Debugflag: *debugFlag, TofilePath: *logFileFlag} controlChan := make(chan bool) @@ -179,29 +178,30 @@ func main() { lg.Info("Clusters loaded") fileChangeSignal := lbConfig.WatchFileChange(controlChan, &wg) - intervalTickerSignal := sleep(DefaultSleepDuration, controlChan,&wg) + intervalTickerSignal := sleep(DefaultSleepDuration, controlChan, &wg) for { select { - case fileWatcherData := <-fileChangeSignal: - if fileWatcherData.IsErrorPresent(){ - // stop all operations - controlChan <- true - return - } - lg.Info("Config Changed") - config, lbclusters, err = lbConfig.Load() - if err != nil { - lg.Error(fmt.Sprintf("Error getting the clusters (something wrong in %v", configFileFlag)) - } - case <-intervalTickerSignal: - checkAliases(config, lg, lbclusters) - break + case fileWatcherData := <-fileChangeSignal: + if fileWatcherData.IsErrorPresent() { + // stop all operations + controlChan <- true + return + } + lg.Info("Config Changed") + config, lbclusters, err = lbConfig.Load() + if err != nil { + lg.Error(fmt.Sprintf("Error getting the clusters (something wrong in %v", configFileFlag)) + } + case <-intervalTickerSignal: + checkAliases(config, lg, lbclusters) + break } } } + // todo: add some tests func checkAliases(config lbconfig.Config, lg lbcluster.Log, lbclusters []lbcluster.LBCluster) { - hostCheckChannel := make(chan lbhost.LBHost) + hostCheckChannel := make(chan lbhost.Host) defer close(hostCheckChannel) hostname, e := os.Hostname() @@ -212,7 +212,7 @@ func checkAliases(config lbconfig.Config, lg lbcluster.Log, lbclusters []lbclust //var wg sync.WaitGroup updateDNS := true lg.Info("Checking if any of the " + strconv.Itoa(len(lbclusters)) + " clusters needs updating") - hostsToCheck := make(map[string]lbhost.LBHost) + hostsToCheck := make(map[string]lbhost.Host) var clustersToUpdate []*lbcluster.LBCluster /* First, let's identify the hosts that have to be checked */ for i := range lbclusters { @@ -227,14 +227,14 @@ func checkAliases(config lbconfig.Config, lg lbcluster.Log, lbclusters []lbclust if len(hostsToCheck) > 0 { /* Now, let's go through the hosts, issuing the snmp call */ for _, hostValue := range hostsToCheck { - go func(myHost lbhost.LBHost) { - myHost.Snmp_req() + go func(myHost lbhost.Host) { + myHost.SNMPDiscovery() hostCheckChannel <- myHost }(hostValue) } lg.Debug("start gathering the results") for hostChanData := range hostCheckChannel { - hostsToCheck[hostChanData.Host_name] = hostChanData + hostsToCheck[hostChanData.GetName()] = hostChanData } lg.Debug("All the hosts have been tested") @@ -245,10 +245,14 @@ func checkAliases(config lbconfig.Config, lg lbcluster.Log, lbclusters []lbclust //todo: try to update clusters in parallel for _, pc := range clustersToUpdate { pc.Write_to_log("DEBUG", "READY TO UPDATE THE CLUSTER") - if pc.FindBestHosts(hostsToCheck) { + isDNSUpdateValid, err := pc.FindBestHosts(hostsToCheck) + if err != nil { + log.Fatalf("Error while finding best hosts. error:%v", err) + } + if isDNSUpdateValid { if updateDNS { pc.Write_to_log("DEBUG", "Should update dns is true") - // todo: try to implement retry mechanism + // todo: try to implement retry mechanismlbcluster/lbcluster_dns.go pc.RefreshDNS(config.GetDNSManager(), config.GetTSIGKeyPrefix(), config.GetTSIGInternalKey(), config.GetTSIGExternalKey()) } else { pc.Write_to_log("DEBUG", "should_update_dns false") diff --git a/lbhost/lbhost.go b/lbhost/lbhost.go index 662546e..82f2a58 100644 --- a/lbhost/lbhost.go +++ b/lbhost/lbhost.go @@ -3,6 +3,8 @@ package lbhost import ( // "encoding/json" "fmt" + "lb-experts/golbd/lbcluster" + //"io/ioutil" "github.com/reguero/go-snmplib" //"math/rand" @@ -20,8 +22,11 @@ import ( "time" ) -const TIMEOUT int = 10 -const OID string = ".1.3.6.1.4.1.96.255.1" +const ( + TIMEOUT int = 10 + OID string = ".1.3.6.1.4.1.96.255.1" + DefaultResponseInt = 100000 +) type LBHostTransportResult struct { Transport string @@ -31,112 +36,93 @@ type LBHostTransportResult struct { Response_error string } type LBHost struct { - Cluster_name string - Host_name string - Host_transports []LBHostTransportResult - Loadbalancing_username string - Loadbalancing_password string - LogFile string - logMu sync.Mutex - Debugflag bool + ClusterConfig lbcluster.Config + Host_name string + HostTransports []LBHostTransportResult + Logger lbcluster.Logger } -// todo: refractor into smaller functions -func (self *LBHost) Snmp_req() { +type Host interface { + GetName() string + SNMPDiscovery() + GetClusterConfig() *lbcluster.Config + GetLoadForAlias(clusterName string) int + GetWorkingIPs() ([]net.IP, error) + GetAllIPs() ([]net.IP, error) + GetIps() ([]net.IP, error) +} - self.find_transports() +func NewLBHost(clusterConfig lbcluster.Config, logger lbcluster.Logger) Host { + return &LBHost{ + ClusterConfig: clusterConfig, + Logger: logger, + } +} + +func (lh *LBHost) GetName() string { + return lh.Host_name +} +func (lh *LBHost) GetClusterConfig() *lbcluster.Config { + return &lh.ClusterConfig +} - for i, my_transport := range self.Host_transports { - my_transport.Response_int = 100000 - transport := my_transport.Transport - node_ip := my_transport.IP.String() +// todo: refractor into smaller functions +func (lh *LBHost) SNMPDiscovery() { + lh.find_transports() + for i, hostTransport := range lh.HostTransports { + hostTransport.Response_int = DefaultResponseInt + node_ip := hostTransport.IP.String() /* There is no need to put square brackets around the ipv6 addresses*/ - self.Write_to_log("DEBUG", "Checking the host "+node_ip+" with "+transport) - snmp, err := snmplib.NewSNMPv3(node_ip, self.Loadbalancing_username, "MD5", self.Loadbalancing_password, "NOPRIV", self.Loadbalancing_password, + lh.Write_to_log("DEBUG", "Checking the host "+node_ip+" with "+hostTransport.Transport) + snmp, err := snmplib.NewSNMPv3(node_ip, lh.ClusterConfig.Loadbalancing_username, "MD5", lh.ClusterConfig.Loadbalancing_password, "NOPRIV", lh.ClusterConfig.Loadbalancing_password, time.Duration(TIMEOUT)*time.Second, 2) if err != nil { - // Failed to create snmpgo.SNMP object - my_transport.Response_error = fmt.Sprintf("contacted node: error creating the snmp object: %v", err) + hostTransport.Response_error = fmt.Sprintf("contacted node: error creating the snmp object: %v", err) } else { defer snmp.Close() err = snmp.Discover() - if err != nil { - my_transport.Response_error = fmt.Sprintf("contacted node: error in the snmp discovery: %v", err) - + hostTransport.Response_error = fmt.Sprintf("contacted node: error in the snmp discovery: %v", err) } else { - - oid, err := snmplib.ParseOid(OID) - - if err != nil { - // Failed to parse Oids - my_transport.Response_error = fmt.Sprintf("contacted node: Error parsing the OID %v", err) - - } else { - pdu, err := snmp.GetV3(oid) - - if err != nil { - my_transport.Response_error = fmt.Sprintf("contacted node: The getv3 gave the following error: %v ", err) - - } else { - - self.Write_to_log("INFO", fmt.Sprintf("contacted node: transport: %v ip: %v - reply was %v", transport, node_ip, pdu)) - - //var pduInteger int - switch t := pdu.(type) { - case int: - my_transport.Response_int = pdu.(int) - case string: - my_transport.Response_string = pdu.(string) - default: - my_transport.Response_error = fmt.Sprintf("The node returned an unexpected type %s in %v", t, pdu) - } - } - } + lh.setTransportResponse(snmp, &hostTransport) } } - self.Host_transports[i] = my_transport + lh.HostTransports[i] = hostTransport } - - self.Write_to_log("DEBUG", "All the ips have been tested") - /*for _, my_transport := range self.Host_transports { - self.Write_to_log("INFO", fmt.Sprintf("%v", my_transport)) - }*/ + lh.Write_to_log("DEBUG", "All the ips have been tested") } -func (self *LBHost) Write_to_log(level string, msg string) error { - var err error - if level == "DEBUG" && !self.Debugflag { - //The debug messages should not appear - return nil - } - if !strings.HasSuffix(msg, "\n") { - msg += "\n" +func (lh *LBHost) setTransportResponse(snmpClient *snmplib.SNMP, lbHostTransportResultPayload *LBHostTransportResult) { + oid, err := snmplib.ParseOid(OID) + if err != nil { + lbHostTransportResultPayload.Response_error = fmt.Sprintf("contacted node: Error parsing the OID %v", err) + return } - timestamp := time.Now().Format(time.StampMilli) - msg = fmt.Sprintf("%s lbd[%d]: %s: cluster: %s node: %s %s", timestamp, os.Getpid(), level, self.Cluster_name, self.Host_name, msg) - - self.logMu.Lock() - defer self.logMu.Unlock() - - f, err := os.OpenFile(self.LogFile, os.O_APPEND|os.O_WRONLY|os.O_CREATE, 0640) + pdu, err := snmpClient.GetV3(oid) if err != nil { - return err + lbHostTransportResultPayload.Response_error = fmt.Sprintf("contacted node: The getv3 gave the following error: %v ", err) + return + } + lh.Write_to_log("INFO", fmt.Sprintf("contacted node: transport: %v ip: %v - reply was %v", lbHostTransportResultPayload.Transport, lbHostTransportResultPayload.IP.String(), pdu)) + switch t := pdu.(type) { + case int: + lbHostTransportResultPayload.Response_int = pdu.(int) + case string: + lbHostTransportResultPayload.Response_string = pdu.(string) + default: + lbHostTransportResultPayload.Response_error = fmt.Sprintf("The node returned an unexpected type %s in %v", t, pdu) } - defer f.Close() - _, err = fmt.Fprintf(f, msg) - - return err } + // todo: instead of polling try adhoc webhook updates -func (self *LBHost) Get_load_for_alias(cluster_name string) int { +func (lh *LBHost) GetLoadForAlias(clusterName string) int { my_load := -200 - for _, my_transport := range self.Host_transports { + for _, my_transport := range lh.HostTransports { pduInteger := my_transport.Response_int - re := regexp.MustCompile(cluster_name + "=([0-9]+)") + re := regexp.MustCompile(clusterName + "=([0-9]+)") submatch := re.FindStringSubmatch(my_transport.Response_string) if submatch != nil { @@ -146,80 +132,75 @@ func (self *LBHost) Get_load_for_alias(cluster_name string) int { if (pduInteger > 0 && pduInteger < my_load) || (my_load < 0) { my_load = pduInteger } - self.Write_to_log("DEBUG", fmt.Sprintf("Possible load is %v", pduInteger)) + lh.Write_to_log("DEBUG", fmt.Sprintf("Possible load is %v", pduInteger)) } - self.Write_to_log("DEBUG", fmt.Sprintf("THE LOAD IS %v, ", my_load)) + lh.Write_to_log("DEBUG", fmt.Sprintf("THE LOAD IS %v, ", my_load)) return my_load } -func (self *LBHost) Get_working_IPs() ([]net.IP, error) { +func (lh *LBHost) GetWorkingIPs() ([]net.IP, error) { var my_ips []net.IP - for _, my_transport := range self.Host_transports { + for _, my_transport := range lh.HostTransports { if (my_transport.Response_int > 0) && (my_transport.Response_error == "") { my_ips = append(my_ips, my_transport.IP) } } - self.Write_to_log("INFO", fmt.Sprintf("The ips for this host are %v", my_ips)) + lh.Write_to_log("INFO", fmt.Sprintf("The ips for this host are %v", my_ips)) return my_ips, nil } -func (self *LBHost) Get_all_IPs() ([]net.IP, error) { +func (lh *LBHost) GetAllIPs() ([]net.IP, error) { var my_ips []net.IP - for _, my_transport := range self.Host_transports { + for _, my_transport := range lh.HostTransports { my_ips = append(my_ips, my_transport.IP) } - self.Write_to_log("INFO", fmt.Sprintf("All ips for this host are %v", my_ips)) + lh.Write_to_log("INFO", fmt.Sprintf("All ips for this host are %v", my_ips)) return my_ips, nil } -func (self *LBHost) Get_Ips() ([]net.IP, error) { - +func (lh *LBHost) GetIps() ([]net.IP, error) { var ips []net.IP - var err error - re := regexp.MustCompile(".*no such host") - net.DefaultResolver.StrictErrors = true - for i := 0; i < 3; i++ { - self.Write_to_log("INFO", "Getting the ip addresses") - ips, err = net.LookupIP(self.Host_name) + lh.Write_to_log("INFO", "Getting the ip addresses") + ips, err = net.LookupIP(lh.Host_name) if err == nil { return ips, nil } - self.Write_to_log("WARNING", fmt.Sprintf("LookupIP: %v has incorrect or missing IP address (%v) ", self.Host_name, err)) + lh.Write_to_log("WARNING", fmt.Sprintf("LookupIP: %v has incorrect or missing IP address (%v) ", lh.Host_name, err)) submatch := re.FindStringSubmatch(err.Error()) if submatch != nil { - self.Write_to_log("INFO", "There is no need to retry this error") + lh.Write_to_log("INFO", "There is no need to retry this error") return nil, err } } - self.Write_to_log("ERROR", "After several retries, we couldn't get the ips!. Let's try with partial results") + lh.Write_to_log("ERROR", "After several retries, we couldn't get the ips!. Let's try with partial results") net.DefaultResolver.StrictErrors = false - ips, err = net.LookupIP(self.Host_name) + ips, err = net.LookupIP(lh.Host_name) if err != nil { - self.Write_to_log("ERROR", fmt.Sprintf("It didn't work :(. This node will be ignored during this evaluation: %v", err)) + lh.Write_to_log("ERROR", fmt.Sprintf("It didn't work :(. This node will be ignored during this evaluation: %v", err)) } return ips, err } -func (self *LBHost) find_transports() { - self.Write_to_log("DEBUG", "Let's find the ips behind this host") +func (lh *LBHost) find_transports() { + lh.Write_to_log("DEBUG", "Let's find the ips behind this host") - ips, _ := self.Get_Ips() + ips, _ := lh.GetIps() for _, ip := range ips { transport := "udp" // If there is an IPv6 address use udp6 transport if ip.To4() == nil { transport = "udp6" } - self.Host_transports = append(self.Host_transports, LBHostTransportResult{Transport: transport, - Response_int: 100000, Response_string: "", IP: ip, + lh.HostTransports = append(lh.HostTransports, LBHostTransportResult{Transport: transport, + Response_int: DefaultResponseInt, Response_string: "", IP: ip, Response_error: ""}) } diff --git a/tests/aliasload_test.go b/tests/aliasload_test.go index bb45d3d..c3d540d 100644 --- a/tests/aliasload_test.go +++ b/tests/aliasload_test.go @@ -7,7 +7,7 @@ import ( "gitlab.cern.ch/lb-experts/golbd/lbhost" ) -//Function TestGetLoadHosts tests the function Get_load_for_alias +//Function TestGetLoadHosts tests the function GetLoadForAlias func TestGetLoadHosts(t *testing.T) { hosts := []lbhost.LBHost{ @@ -18,9 +18,9 @@ func TestGetLoadHosts(t *testing.T) { } expectedhost0 := hosts[0].Host_transports[0].Response_int - //expectedhost1 := hosts[1].Host_transports[0].Response_int + //expectedhost1 := hosts[1].HostTransports[0].Response_int expectedhost2 := hosts[2].Host_transports[0].Response_int - //expectedhost3 := hosts[3].Host_transports[0].Response_int + //expectedhost3 := hosts[3].HostTransports[0].Response_int if !reflect.DeepEqual(hosts[0].Get_load_for_alias(hosts[0].Cluster_name), expectedhost0) { t.Errorf(" got\n%v\nexpected\n%v", hosts[0].Get_load_for_alias(hosts[0].Cluster_name), expectedhost0) diff --git a/tests/file_watch_test.go b/tests/file_watch_test.go deleted file mode 100644 index 05c0b06..0000000 --- a/tests/file_watch_test.go +++ /dev/null @@ -1,2 +0,0 @@ -package main_test - From 1ebb486dd19461df021a17b3f46ab099274107ba Mon Sep 17 00:00:00 2001 From: Yadu Nandan Date: Sat, 7 May 2022 22:08:17 -0700 Subject: [PATCH 05/20] refractored logging with tests --- lbcluster/lbcluster.go | 39 +++++----- lbcluster/lbcluster_dns.go | 23 +++--- lbcluster/lbcluster_log.go | 149 ++++++++++++++++++------------------ lbd.go | 19 +++-- tests/lbcluster_log_test.go | 120 +++++++++++++++++++++++++++++ 5 files changed, 235 insertions(+), 115 deletions(-) create mode 100644 tests/lbcluster_log_test.go diff --git a/lbcluster/lbcluster.go b/lbcluster/lbcluster.go index d01b94f..4bae265 100644 --- a/lbcluster/lbcluster.go +++ b/lbcluster/lbcluster.go @@ -32,7 +32,7 @@ type LBCluster struct { Current_best_ips []net.IP Previous_best_ips_dns []net.IP Current_index int - Slog *Log + Slog Logger } type Config struct { @@ -98,7 +98,7 @@ func (lbc *LBCluster) Time_to_refresh() bool { //Get_list_hosts Get the hosts for an alias func (lbc *LBCluster) Get_list_hosts(current_list map[string]lbhost.Host) { - lbc.Write_to_log("DEBUG", "Getting the list of hosts for the alias") + lbc.Slog.Debug("Getting the list of hosts for the alias") for host := range lbc.Host_metric_table { myHost, ok := current_list[host] if ok { @@ -141,7 +141,7 @@ func (lbc *LBCluster) FindBestHosts(hosts_to_check map[string]lbhost.Host) (bool _, ok := allMetrics[lbc.Parameters.Metric] if !ok { - lbc.Write_to_log("ERROR", "wrong parameter(metric) in definition of cluster "+lbc.Parameters.Metric) + lbc.Slog.Error("wrong parameter(metric) in definition of cluster " + lbc.Parameters.Metric) return false, nil } lbc.Time_of_last_evaluation = time.Now() @@ -156,13 +156,13 @@ func (lbc *LBCluster) FindBestHosts(hosts_to_check map[string]lbhost.Host) (bool if len(lbc.Current_best_ips) == 0 { nodes = "NONE" } - lbc.Write_to_log("INFO", "best hosts are: "+nodes) - return true, nil + lbc.Slog.Info("best hosts are: " + nodes) + return true,nil } // ApplyMetric This is the core of the lbcluster: based on the metrics, select the best hosts -func (lbc *LBCluster) ApplyMetric(hosts_to_check map[string]lbhost.Host) (bool, error) { - lbc.Write_to_log("INFO", "Got metric = "+lbc.Parameters.Metric) +func (lbc *LBCluster) ApplyMetric(hosts_to_check map[string]lbhost.Host) (bool,error) { + lbc.Slog.Info("Got metric = " + lbc.Parameters.Metric) pl := make(NodeList, len(lbc.Host_metric_table)) i := 0 for _, v := range lbc.Host_metric_table { @@ -175,7 +175,7 @@ func (lbc *LBCluster) ApplyMetric(hosts_to_check map[string]lbhost.Host) (bool, return false, err } sort.Sort(pl) - lbc.Write_to_log("DEBUG", fmt.Sprintf("%v", pl)) + lbc.Slog.Debug(fmt.Sprintf("%v", pl)) var sorted_host_list []Node var useful_host_list []Node for _, v := range pl { @@ -184,7 +184,7 @@ func (lbc *LBCluster) ApplyMetric(hosts_to_check map[string]lbhost.Host) (bool, } sorted_host_list = append(sorted_host_list, v) } - lbc.Write_to_log("DEBUG", fmt.Sprintf("%v", useful_host_list)) + lbc.Slog.Debug(fmt.Sprintf("%v", useful_host_list)) useful_hosts := len(useful_host_list) listLength := len(pl) max := lbc.Parameters.Best_hosts @@ -192,17 +192,17 @@ func (lbc *LBCluster) ApplyMetric(hosts_to_check map[string]lbhost.Host) (bool, max = listLength } if max > listLength { - lbc.Write_to_log("WARNING", fmt.Sprintf("impossible to return %v hosts from the list of %v hosts (%v). Check the configuration of cluster. Returning %v hosts.", + lbc.Slog.Warning(fmt.Sprintf("impossible to return %v hosts from the list of %v hosts (%v). Check the configuration of cluster. Returning %v hosts.", max, listLength, lbc.concatenateNodes(sorted_host_list), listLength)) max = listLength } lbc.Current_best_ips = []net.IP{} if listLength == 0 { - lbc.Write_to_log("ERROR", "cluster has no hosts defined ! Check the configuration.") + lbc.Slog.Error("cluster has no hosts defined ! Check the configuration.") } else if useful_hosts == 0 { if lbc.Parameters.Metric == "minimum" { - lbc.Write_to_log("WARNING", fmt.Sprintf("no usable hosts found for cluster! Returning random %v hosts.", max)) + lbc.Slog.Warning(fmt.Sprintf("no usable hosts found for cluster! Returning random %v hosts.", max)) //Get hosts with all IPs even when not OK for SNMP lbc.ReEvaluateHostsForMinimum(hosts_to_check) i := 0 @@ -218,17 +218,17 @@ func (lbc *LBCluster) ApplyMetric(hosts_to_check map[string]lbhost.Host) (bool, for i := 0; i < max; i++ { lbc.Current_best_ips = append(lbc.Current_best_ips, pl[i].IPs...) } - lbc.Write_to_log("WARNING", fmt.Sprintf("We have put random hosts behind the alias: %v", lbc.Current_best_ips)) + lbc.Slog.Warning(fmt.Sprintf("We have put random hosts behind the alias: %v", lbc.Current_best_ips)) } else if (lbc.Parameters.Metric == "minino") || (lbc.Parameters.Metric == "cmsweb") { - lbc.Write_to_log("WARNING", "no usable hosts found for cluster! Returning no hosts.") + lbc.Slog.Warning("no usable hosts found for cluster! Returning no hosts.") } else if lbc.Parameters.Metric == "cmsfrontier" { - lbc.Write_to_log("WARNING", "no usable hosts found for cluster! Skipping the DNS update") - return false, nil + lbc.Slog.Warning("no usable hosts found for cluster! Skipping the DNS update") + return false,nil } } else { if useful_hosts < max { - lbc.Write_to_log("WARNING", fmt.Sprintf("only %v useable hosts found in cluster", useful_hosts)) + lbc.Slog.Warning(fmt.Sprintf("only %v useable hosts found in cluster", useful_hosts)) max = useful_hosts } for i := 0; i < max; i++ { @@ -300,14 +300,13 @@ func (lbc *LBCluster) EvaluateHosts(hostsToCheck map[string]lbhost.Host) { for currenthost := range lbc.Host_metric_table { host := hostsToCheck[currenthost] - //todo: parallelize here ips, err := host.GetWorkingIPs() if err != nil { ips, err = host.GetIps() } lbc.Host_metric_table[currenthost] = Node{host.GetLoadForAlias(lbc.ClusterConfig.Cluster_name), ips} - lbc.Write_to_log("DEBUG", fmt.Sprintf("node: %s It has a load of %d", currenthost, lbc.Host_metric_table[currenthost].Load)) + lbc.Slog.Debug(fmt.Sprintf("node: %s It has a load of %d", currenthost, lbc.Host_metric_table[currenthost].Load)) } } @@ -321,6 +320,6 @@ func (lbc *LBCluster) ReEvaluateHostsForMinimum(hostsToCheck map[string]lbhost.H ips, err = host.GetIps() } lbc.Host_metric_table[currenthost] = Node{host.GetLoadForAlias(lbc.ClusterConfig.Cluster_name), ips} - lbc.Write_to_log("DEBUG", fmt.Sprintf("node: %s It has a load of %d", currenthost, lbc.Host_metric_table[currenthost].Load)) + lbc.Slog.Debug(fmt.Sprintf("node: %s It has a load of %d", currenthost, lbc.Host_metric_table[currenthost].Load)) } } diff --git a/lbcluster/lbcluster_dns.go b/lbcluster/lbcluster_dns.go index 9d0ddbf..3c01f4b 100644 --- a/lbcluster/lbcluster_dns.go +++ b/lbcluster/lbcluster_dns.go @@ -14,26 +14,26 @@ func (lbc *LBCluster) RefreshDNS(dnsManager, keyPrefix, internalKey, externalKey e := lbc.GetStateDNS(dnsManager) if e != nil { - lbc.Write_to_log("WARNING", fmt.Sprintf("Get_state_dns Error: %v", e.Error())) + lbc.Slog.Warning(fmt.Sprintf("Get_state_dns Error: %v", e.Error())) } pbiDNS := lbc.concatenateIps(lbc.Previous_best_ips_dns) cbi := lbc.concatenateIps(lbc.Current_best_ips) if pbiDNS == cbi { - lbc.Write_to_log("INFO", fmt.Sprintf("DNS not update keyName %v cbh == pbhDns == %v", keyPrefix, cbi)) + lbc.Slog.Info(fmt.Sprintf("DNS not update keyName %v cbh == pbhDns == %v", keyPrefix, cbi)) return } - lbc.Write_to_log("INFO", fmt.Sprintf("Updating the DNS with %v (previous state was %v)", cbi, pbiDNS)) + lbc.Slog.Info(fmt.Sprintf("Updating the DNS with %v (previous state was %v)", cbi, pbiDNS)) e = lbc.updateDNS(keyPrefix+"internal.", internalKey, dnsManager) if e != nil { - lbc.Write_to_log("WARNING", fmt.Sprintf("Internal Update_dns Error: %v", e.Error())) + lbc.Slog.Warning(fmt.Sprintf("Internal Update_dns Error: %v", e.Error())) } if lbc.externallyVisible() { e = lbc.updateDNS(keyPrefix+"external.", externalKey, dnsManager) if e != nil { - lbc.Write_to_log("WARNING", fmt.Sprintf("External Update_dns Error: %v", e.Error())) + lbc.Slog.Warning(fmt.Sprintf("External Update_dns Error: %v", e.Error())) } } } @@ -67,17 +67,16 @@ func (lbc *LBCluster) updateDNS(keyName, tsigKey, dnsManager string) error { } m.Insert([]dns.RR{rrInsert}) } - lbc.Write_to_log("INFO", fmt.Sprintf("WE WOULD UPDATE THE DNS WITH THE IPS %v", m)) + lbc.Slog.Info(fmt.Sprintf("WE WOULD UPDATE THE DNS WITH THE IPS %v", m)) c := new(dns.Client) m.SetTsig(keyName, dns.HmacMD5, 300, time.Now().Unix()) c.TsigSecret = map[string]string{keyName: tsigKey} _, _, err := c.Exchange(m, dnsManager+":53") if err != nil { - // todo: consider retries here - lbc.Write_to_log("ERROR", fmt.Sprintf("DNS update failed with (%v)", err)) + lbc.Slog.Error(fmt.Sprintf("DNS update failed with (%v)", err)) return err } - lbc.Write_to_log("INFO", fmt.Sprintf("DNS update with keyName %v", keyName)) + lbc.Slog.Info(fmt.Sprintf("DNS update with keyName %v", keyName)) return nil } @@ -86,7 +85,7 @@ func (lbc *LBCluster) getIpsFromDNS(m *dns.Msg, dnsManager string, dnsType uint1 m.SetQuestion(lbc.ClusterConfig.Cluster_name+".", dnsType) in, err := dns.Exchange(m, dnsManager+":53") if err != nil { - lbc.Write_to_log("ERROR", fmt.Sprintf("Error getting the ipv4 state of dns: %v", err)) + lbc.Slog.Error(fmt.Sprintf("Error getting the ipv4 state of dns: %v", err)) return err } for _, a := range in.Answer { @@ -105,7 +104,7 @@ func (lbc *LBCluster) GetStateDNS(dnsManager string) error { m := new(dns.Msg) var ips []net.IP m.SetEdns0(4096, false) - lbc.Write_to_log("DEBUG", "Getting the ips from the DNS") + lbc.Slog.Debug("Getting the ips from the DNS") err := lbc.getIpsFromDNS(m, dnsManager, dns.TypeA, &ips) if err != nil { @@ -116,7 +115,7 @@ func (lbc *LBCluster) GetStateDNS(dnsManager string) error { return err } - lbc.Write_to_log("INFO", fmt.Sprintf("Let's keep the list of ips : %v", ips)) + lbc.Slog.Info(fmt.Sprintf("Let's keep the list of ips : %v", ips)) lbc.Previous_best_ips_dns = ips return nil diff --git a/lbcluster/lbcluster_log.go b/lbcluster/lbcluster_log.go index 3df1046..63652c7 100644 --- a/lbcluster/lbcluster_log.go +++ b/lbcluster/lbcluster_log.go @@ -9,101 +9,102 @@ import ( "time" ) +const ( + DefaultLbdTag = "lbd" + logLevelInfo = "INFO" + logLevelDebug = "DEBUG" + logLevelWarning = "WARNING" + logLevelError = "ERROR" +) + +func NewLoggerFactory(logFilePath string) (Logger, error) { + log, err := syslog.New(syslog.LOG_NOTICE, DefaultLbdTag) + if err != nil { + return nil, err + } + if strings.EqualFold(logFilePath, "") { + return nil, fmt.Errorf("empty log file path") + } + _, err = os.OpenFile(logFilePath, os.O_APPEND|os.O_WRONLY|os.O_CREATE, 0640) + if err != nil { + return nil, err + } + return &Log{ + logWriter: log, + filePath: logFilePath}, + nil +} + //Log struct for the log type Log struct { - SyslogWriter *syslog.Writer - Stdout bool - Debugflag bool - TofilePath string - logMu sync.Mutex + logWriter *syslog.Writer + shouldWriteToSTD bool + isDebugAllowed bool + filePath string + logMu sync.Mutex } -// todo: refractor logging and consider log based snapshots -// todo: consider start and end times of dns updates //Logger struct for the Logger interface type Logger interface { - Info(s string) error - Warning(s string) error - Debug(s string) error - Error(s string) error + EnableDebugMode() + EnableWriteToSTd() + Info(s string) + Warning(s string) + Debug(s string) + Error(s string) } -//Write_to_log put something in the log file -func (lbc *LBCluster) Write_to_log(level string, msg string) error { - - myMessage := "cluster: " + lbc.ClusterConfig.Cluster_name + " " + msg - - if level == "INFO" { - lbc.Slog.Info(myMessage) - } else if level == "DEBUG" { - lbc.Slog.Debug(myMessage) - } else if level == "WARNING" { - lbc.Slog.Warning(myMessage) - } else if level == "ERROR" { - lbc.Slog.Error(myMessage) - } else { - lbc.Slog.Error("LEVEL " + level + " NOT UNDERSTOOD, ASSUMING ERROR " + myMessage) - } +func (l *Log) EnableDebugMode() { + l.isDebugAllowed = true +} - return nil +func (l *Log) EnableWriteToSTd() { + l.shouldWriteToSTD = true } //Info write as Info -func (l *Log) Info(s string) error { - var err error - if l.SyslogWriter != nil { - err = l.SyslogWriter.Info(s) +func (l *Log) Info(s string) { + if l.logWriter != nil { + _ = l.logWriter.Info(s) } - if l.Stdout || (l.TofilePath != "") { - err = l.writefilestd("INFO: " + s) + if l.shouldWriteToSTD || (l.filePath != "") { + l.write(fmt.Sprintf("%s: %s", logLevelInfo, s)) } - return err - } //Warning write as Warning -func (l *Log) Warning(s string) error { - var err error - if l.SyslogWriter != nil { - err = l.SyslogWriter.Warning(s) +func (l *Log) Warning(s string) { + if l.logWriter != nil { + _ = l.logWriter.Warning(s) } - if l.Stdout || (l.TofilePath != "") { - err = l.writefilestd("WARNING: " + s) + if l.shouldWriteToSTD || (l.filePath != "") { + l.write(fmt.Sprintf("%s: %s", logLevelWarning, s)) } - return err - } //Debug write as Debug -func (l *Log) Debug(s string) error { - var err error - if l.Debugflag { - if l.SyslogWriter != nil { - err = l.SyslogWriter.Debug(s) +func (l *Log) Debug(s string) { + if l.isDebugAllowed { + if l.logWriter != nil { + _ = l.logWriter.Debug(s) } - if l.Stdout || (l.TofilePath != "") { - err = l.writefilestd("DEBUG: " + s) + if l.shouldWriteToSTD || (l.filePath != "") { + l.write(fmt.Sprintf("%s: %s", logLevelDebug, s)) } } - return err - } //Error write as Error -func (l *Log) Error(s string) error { - var err error - if l.SyslogWriter != nil { - err = l.SyslogWriter.Err(s) +func (l *Log) Error(s string) { + if l.logWriter != nil { + _ = l.logWriter.Err(s) } - if l.Stdout || (l.TofilePath != "") { - err = l.writefilestd("ERROR: " + s) + if l.shouldWriteToSTD || (l.filePath != "") { + l.write(fmt.Sprintf("%s: %s", logLevelError, s)) } - return err - } -func (l *Log) writefilestd(s string) error { - var err error +func (l *Log) write(s string) { tag := "lbd" nl := "" if !strings.HasSuffix(s, "\n") { @@ -115,16 +116,18 @@ func (l *Log) writefilestd(s string) error { tag, os.Getpid(), s, nl) l.logMu.Lock() defer l.logMu.Unlock() - if l.Stdout { - _, err = fmt.Printf(msg) + if l.shouldWriteToSTD { + fmt.Printf(msg) } - if l.TofilePath != "" { - f, err := os.OpenFile(l.TofilePath, os.O_APPEND|os.O_WRONLY|os.O_CREATE, 0640) - if err != nil { - return err - } - defer f.Close() - _, err = fmt.Fprintf(f, msg) + + f, err := os.OpenFile(l.filePath, os.O_APPEND|os.O_WRONLY|os.O_CREATE, 0640) + if err != nil { + _, _ = fmt.Fprintf(os.Stderr, "error while opening the log file. error: %v", err) + return + } + defer f.Close() + _, err = fmt.Fprintf(f, msg) + if err != nil { + _, _ = fmt.Fprintf(os.Stderr, "error while writing to the log file. error: %v", err) } - return err } diff --git a/lbd.go b/lbd.go index aa95bd9..7f14850 100644 --- a/lbd.go +++ b/lbd.go @@ -50,7 +50,7 @@ type ConfigFileChangeSignal struct { readError error } -func shouldUpdateDNS(config lbconfig.Config, hostname string, lg *lbcluster.Log) bool { +func shouldUpdateDNS(config lbconfig.Config, hostname string, lg lbcluster.Logger) bool { if strings.EqualFold(hostname, config.GetMasterHost()) { return true } @@ -91,7 +91,7 @@ func shouldUpdateDNS(config lbconfig.Config, hostname string, lg *lbcluster.Log) } -func updateHeartbeat(config lbconfig.Config, hostname string, lg *lbcluster.Log) error { +func updateHeartbeat(config lbconfig.Config, hostname string, lg lbcluster.Logger) error { if hostname != config.GetMasterHost() { return nil } @@ -200,7 +200,7 @@ func main() { } // todo: add some tests -func checkAliases(config lbconfig.Config, lg lbcluster.Log, lbclusters []lbcluster.LBCluster) { +func checkAliases(config lbconfig.Config, lg lbcluster.Logger, lbclusters []lbcluster.LBCluster) { hostCheckChannel := make(chan lbhost.Host) defer close(hostCheckChannel) @@ -217,9 +217,9 @@ func checkAliases(config lbconfig.Config, lg lbcluster.Log, lbclusters []lbclust /* First, let's identify the hosts that have to be checked */ for i := range lbclusters { currentCluster := &lbclusters[i] - currentCluster.Write_to_log("DEBUG", "DO WE HAVE TO UPDATE?") + lg.Debug("DO WE HAVE TO UPDATE?") if currentCluster.Time_to_refresh() { - currentCluster.Write_to_log("INFO", "Time to refresh the cluster") + lg.Info("Time to refresh the cluster") currentCluster.Get_list_hosts(hostsToCheck) clustersToUpdate = append(clustersToUpdate, currentCluster) } @@ -242,23 +242,22 @@ func checkAliases(config lbconfig.Config, lg lbcluster.Log, lbclusters []lbclust updateDNS = shouldUpdateDNS(config, hostname, &lg) /* Finally, let's go through the aliases, selecting the best hosts*/ - //todo: try to update clusters in parallel for _, pc := range clustersToUpdate { - pc.Write_to_log("DEBUG", "READY TO UPDATE THE CLUSTER") + lg.Debug("READY TO UPDATE THE CLUSTER") isDNSUpdateValid, err := pc.FindBestHosts(hostsToCheck) if err != nil { log.Fatalf("Error while finding best hosts. error:%v", err) } if isDNSUpdateValid { if updateDNS { - pc.Write_to_log("DEBUG", "Should update dns is true") + lg.Debug( "Should update dns is true") // todo: try to implement retry mechanismlbcluster/lbcluster_dns.go pc.RefreshDNS(config.GetDNSManager(), config.GetTSIGKeyPrefix(), config.GetTSIGInternalKey(), config.GetTSIGExternalKey()) } else { - pc.Write_to_log("DEBUG", "should_update_dns false") + lg.Debug( "should_update_dns false") } } else { - pc.Write_to_log("DEBUG", "FindBestHosts false") + lg.Debug( "FindBestHosts false") } } } diff --git a/tests/lbcluster_log_test.go b/tests/lbcluster_log_test.go new file mode 100644 index 0000000..d343b18 --- /dev/null +++ b/tests/lbcluster_log_test.go @@ -0,0 +1,120 @@ +package main_test + +import ( + "io/ioutil" + "lb-experts/golbd/lbcluster" + "os" + "strings" + "testing" +) + +const testLogFilePath = "../tests/sample.log" + +func TestLBClusterLoggerForInitFailure(t *testing.T) { + _, err := lbcluster.NewLoggerFactory("") + if err==nil { + t.Errorf("expected error not thrown") + } +} + +func TestLBClusterLoggerForInitSuccess(t *testing.T) { + defer deleteFile(t, testLogFilePath) + logger, err := lbcluster.NewLoggerFactory(testLogFilePath) + if err!=nil { + t.Fail() + t.Errorf("unexpected error thrown. error: %v",err) + } + if logger==nil{ + t.Fail() + t.Errorf("logger instance is nil") + } +} + +func TestLBClusterLoggerForDebugDisabled(t *testing.T) { + defer deleteFile(t, testLogFilePath) + logger, err := lbcluster.NewLoggerFactory(testLogFilePath) + if err!=nil { + t.Fail() + t.Errorf("unexpected error thrown. error: %v",err) + } + logger.Debug("sample info") + if isLogPresentInFile(t, testLogFilePath, "sample info") { + t.Fail() + t.Errorf("log file does not contain the expected debug info. Expected Info: %s", "sample info") + } +} + +func TestLBClusterLoggerForDebugEnabled(t *testing.T) { + defer deleteFile(t, testLogFilePath) + logger, err := lbcluster.NewLoggerFactory(testLogFilePath) + if err!=nil { + t.Fail() + t.Errorf("unexpected error thrown. error: %v",err) + } + logger.EnableDebugMode() + logger.Debug("sample info") + if !isLogPresentInFile(t, testLogFilePath, "sample info") { + t.Fail() + t.Errorf("log file does not contain the expected debug info. Expected Info: %s", "sample info") + } +} + +func TestLBClusterLoggerForInfo(t *testing.T) { + defer deleteFile(t, testLogFilePath) + logger, err := lbcluster.NewLoggerFactory(testLogFilePath) + if err!=nil { + t.Fail() + t.Errorf("unexpected error thrown. error: %v",err) + } + logger.Info("sample info") + if !isLogPresentInFile(t, testLogFilePath, "INFO: sample info") { + t.Fail() + t.Errorf("log file does not contain the expected debug info. Expected Info: %s", "INFO: sample info") + } +} + +func TestLBClusterLoggerForWarning(t *testing.T) { + defer deleteFile(t, testLogFilePath) + logger, err := lbcluster.NewLoggerFactory(testLogFilePath) + if err!=nil { + t.Fail() + t.Errorf("unexpected error thrown. error: %v",err) + } + logger.Warning("sample info") + if !isLogPresentInFile(t, testLogFilePath, "WARNING: sample info") { + t.Fail() + t.Errorf("log file does not contain the expected debug info. Expected Info: %s", "WARNING: sample info") + } +} + +func TestLBClusterLoggerForError(t *testing.T) { + defer deleteFile(t, testLogFilePath) + logger, err := lbcluster.NewLoggerFactory(testLogFilePath) + if err!=nil { + t.Fail() + t.Errorf("unexpected error thrown. error: %v",err) + } + logger.Error("sample info") + if !isLogPresentInFile(t, testLogFilePath, "ERROR: sample info") { + t.Fail() + t.Errorf("log file does not contain the expected debug info. Expected Info: %s", "ERROR: sample info") + } +} + +func deleteFile(t *testing.T, filePath string){ + err := os.Remove(filePath) + if err!=nil{ + t.Errorf("error whil deleting log file error: %v", err) + } +} + +func isLogPresentInFile(t *testing.T, filePath string, stringToCheck string) bool{ + data, err := ioutil.ReadFile(filePath) + if err!=nil { + t.Errorf("error while reading log file. error: %v", err) + return false + } + return strings.Contains(string(data),stringToCheck) +} + + From 5ef882807fcd2996a94938ee1622a20af29b1da6 Mon Sep 17 00:00:00 2001 From: Yadu Nandan Date: Wed, 11 May 2022 02:21:46 -0700 Subject: [PATCH 06/20] more refractor and fixes --- lbcluster/lbcluster.go | 12 ++++++------ lbconfig/config.go | 5 +++-- lbd.go | 6 +++--- lbhost/lbhost.go | 43 ++++++++++++++++-------------------------- 4 files changed, 28 insertions(+), 38 deletions(-) diff --git a/lbcluster/lbcluster.go b/lbcluster/lbcluster.go index 4bae265..123de8a 100644 --- a/lbcluster/lbcluster.go +++ b/lbcluster/lbcluster.go @@ -4,14 +4,14 @@ import ( "encoding/json" "fmt" "io/ioutil" - "lb-experts/golbd/lbhost" "math/rand" "net" "net/http" - "strings" - "sort" + "strings" "time" + + "lb-experts/golbd/lbhost" ) //WorstValue worst possible load @@ -157,11 +157,11 @@ func (lbc *LBCluster) FindBestHosts(hosts_to_check map[string]lbhost.Host) (bool nodes = "NONE" } lbc.Slog.Info("best hosts are: " + nodes) - return true,nil + return true, nil } // ApplyMetric This is the core of the lbcluster: based on the metrics, select the best hosts -func (lbc *LBCluster) ApplyMetric(hosts_to_check map[string]lbhost.Host) (bool,error) { +func (lbc *LBCluster) ApplyMetric(hosts_to_check map[string]lbhost.Host) (bool, error) { lbc.Slog.Info("Got metric = " + lbc.Parameters.Metric) pl := make(NodeList, len(lbc.Host_metric_table)) i := 0 @@ -224,7 +224,7 @@ func (lbc *LBCluster) ApplyMetric(hosts_to_check map[string]lbhost.Host) (bool,e lbc.Slog.Warning("no usable hosts found for cluster! Returning no hosts.") } else if lbc.Parameters.Metric == "cmsfrontier" { lbc.Slog.Warning("no usable hosts found for cluster! Skipping the DNS update") - return false,nil + return false, nil } } else { if useful_hosts < max { diff --git a/lbconfig/config.go b/lbconfig/config.go index e405ce6..5d7b466 100644 --- a/lbconfig/config.go +++ b/lbconfig/config.go @@ -5,13 +5,14 @@ import ( "encoding/json" "fmt" "io" - "lb-experts/golbd/lbcluster" "net" "os" "strconv" "strings" "sync" "time" + + "lb-experts/golbd/lbcluster" ) const ( @@ -264,7 +265,7 @@ func (c *LBConfig) loadClusters() ([]lbcluster.LBCluster, error) { } lbc.Host_metric_table = hm lbcs = append(lbcs, lbc) - lbc.Write_to_log("INFO", "(re-)loaded cluster ") + lbc.Slog.Info("(re-)loaded cluster ") } else { c.lbLog.Warning("cluster: " + k + " missing parameters for cluster; ignoring the cluster, please check the configuration file " + c.configFilePath) diff --git a/lbd.go b/lbd.go index 7f14850..16a092b 100644 --- a/lbd.go +++ b/lbd.go @@ -250,14 +250,14 @@ func checkAliases(config lbconfig.Config, lg lbcluster.Logger, lbclusters []lbcl } if isDNSUpdateValid { if updateDNS { - lg.Debug( "Should update dns is true") + lg.Debug("Should update dns is true") // todo: try to implement retry mechanismlbcluster/lbcluster_dns.go pc.RefreshDNS(config.GetDNSManager(), config.GetTSIGKeyPrefix(), config.GetTSIGInternalKey(), config.GetTSIGExternalKey()) } else { - lg.Debug( "should_update_dns false") + lg.Debug("should_update_dns false") } } else { - lg.Debug( "FindBestHosts false") + lg.Debug("FindBestHosts false") } } } diff --git a/lbhost/lbhost.go b/lbhost/lbhost.go index 82f2a58..0c824f2 100644 --- a/lbhost/lbhost.go +++ b/lbhost/lbhost.go @@ -1,25 +1,14 @@ package lbhost import ( - // "encoding/json" "fmt" - "lb-experts/golbd/lbcluster" - - //"io/ioutil" - "github.com/reguero/go-snmplib" - //"math/rand" "net" - "os" "regexp" "strconv" - "strings" - "sync" - - // "net/http" - - // "sort" - // "strings" "time" + + "github.com/reguero/go-snmplib" + "lb-experts/golbd/lbcluster" ) const ( @@ -73,7 +62,7 @@ func (lh *LBHost) SNMPDiscovery() { hostTransport.Response_int = DefaultResponseInt node_ip := hostTransport.IP.String() /* There is no need to put square brackets around the ipv6 addresses*/ - lh.Write_to_log("DEBUG", "Checking the host "+node_ip+" with "+hostTransport.Transport) + lh.Logger.Debug("Checking the host " + node_ip + " with " + hostTransport.Transport) snmp, err := snmplib.NewSNMPv3(node_ip, lh.ClusterConfig.Loadbalancing_username, "MD5", lh.ClusterConfig.Loadbalancing_password, "NOPRIV", lh.ClusterConfig.Loadbalancing_password, time.Duration(TIMEOUT)*time.Second, 2) if err != nil { @@ -90,7 +79,7 @@ func (lh *LBHost) SNMPDiscovery() { lh.HostTransports[i] = hostTransport } - lh.Write_to_log("DEBUG", "All the ips have been tested") + lh.Logger.Debug("All the ips have been tested") } func (lh *LBHost) setTransportResponse(snmpClient *snmplib.SNMP, lbHostTransportResultPayload *LBHostTransportResult) { @@ -104,7 +93,7 @@ func (lh *LBHost) setTransportResponse(snmpClient *snmplib.SNMP, lbHostTransport lbHostTransportResultPayload.Response_error = fmt.Sprintf("contacted node: The getv3 gave the following error: %v ", err) return } - lh.Write_to_log("INFO", fmt.Sprintf("contacted node: transport: %v ip: %v - reply was %v", lbHostTransportResultPayload.Transport, lbHostTransportResultPayload.IP.String(), pdu)) + lh.Logger.Info(fmt.Sprintf("contacted node: transport: %v ip: %v - reply was %v", lbHostTransportResultPayload.Transport, lbHostTransportResultPayload.IP.String(), pdu)) switch t := pdu.(type) { case int: lbHostTransportResultPayload.Response_int = pdu.(int) @@ -132,10 +121,10 @@ func (lh *LBHost) GetLoadForAlias(clusterName string) int { if (pduInteger > 0 && pduInteger < my_load) || (my_load < 0) { my_load = pduInteger } - lh.Write_to_log("DEBUG", fmt.Sprintf("Possible load is %v", pduInteger)) + lh.Logger.Debug(fmt.Sprintf("Possible load is %v", pduInteger)) } - lh.Write_to_log("DEBUG", fmt.Sprintf("THE LOAD IS %v, ", my_load)) + lh.Logger.Debug(fmt.Sprintf("THE LOAD IS %v, ", my_load)) return my_load } @@ -148,7 +137,7 @@ func (lh *LBHost) GetWorkingIPs() ([]net.IP, error) { } } - lh.Write_to_log("INFO", fmt.Sprintf("The ips for this host are %v", my_ips)) + lh.Logger.Info(fmt.Sprintf("The ips for this host are %v", my_ips)) return my_ips, nil } @@ -157,7 +146,7 @@ func (lh *LBHost) GetAllIPs() ([]net.IP, error) { for _, my_transport := range lh.HostTransports { my_ips = append(my_ips, my_transport.IP) } - lh.Write_to_log("INFO", fmt.Sprintf("All ips for this host are %v", my_ips)) + lh.Logger.Info(fmt.Sprintf("All ips for this host are %v", my_ips)) return my_ips, nil } @@ -167,30 +156,30 @@ func (lh *LBHost) GetIps() ([]net.IP, error) { re := regexp.MustCompile(".*no such host") net.DefaultResolver.StrictErrors = true for i := 0; i < 3; i++ { - lh.Write_to_log("INFO", "Getting the ip addresses") + lh.Logger.Info("Getting the ip addresses") ips, err = net.LookupIP(lh.Host_name) if err == nil { return ips, nil } - lh.Write_to_log("WARNING", fmt.Sprintf("LookupIP: %v has incorrect or missing IP address (%v) ", lh.Host_name, err)) + lh.Logger.Info(fmt.Sprintf("LookupIP: %v has incorrect or missing IP address (%v) ", lh.Host_name, err)) submatch := re.FindStringSubmatch(err.Error()) if submatch != nil { - lh.Write_to_log("INFO", "There is no need to retry this error") + lh.Logger.Info("There is no need to retry this error") return nil, err } } - lh.Write_to_log("ERROR", "After several retries, we couldn't get the ips!. Let's try with partial results") + lh.Logger.Error("After several retries, we couldn't get the ips!. Let's try with partial results") net.DefaultResolver.StrictErrors = false ips, err = net.LookupIP(lh.Host_name) if err != nil { - lh.Write_to_log("ERROR", fmt.Sprintf("It didn't work :(. This node will be ignored during this evaluation: %v", err)) + lh.Logger.Error(fmt.Sprintf("It didn't work :(. This node will be ignored during this evaluation: %v", err)) } return ips, err } func (lh *LBHost) find_transports() { - lh.Write_to_log("DEBUG", "Let's find the ips behind this host") + lh.Logger.Debug("Let's find the ips behind this host") ips, _ := lh.GetIps() for _, ip := range ips { From 87341ab6927cd1c1f61ff90cd5d55c52cab4e356 Mon Sep 17 00:00:00 2001 From: Yadu Nandan Date: Sun, 8 May 2022 01:46:16 -0700 Subject: [PATCH 07/20] added snapshot support --- lbcluster/lbcluster_log.go | 98 ++++++++++++++++++----- tests/lbcluster_log_test.go | 154 +++++++++++++++++++++++++----------- 2 files changed, 185 insertions(+), 67 deletions(-) diff --git a/lbcluster/lbcluster_log.go b/lbcluster/lbcluster_log.go index 63652c7..3ee4732 100644 --- a/lbcluster/lbcluster_log.go +++ b/lbcluster/lbcluster_log.go @@ -4,6 +4,7 @@ import ( "fmt" "log/syslog" "os" + "regexp" "strings" "sync" "time" @@ -17,6 +18,32 @@ const ( logLevelError = "ERROR" ) +//Log struct for the log +type Log struct { + logWriter *syslog.Writer + shouldWriteToSTD bool + isDebugAllowed bool + filePath string + logMu sync.Mutex + logStartTime time.Time + isSnapShotEnabled bool + snapShotCycleTime time.Duration + logFileBasePath string + logFileExtension string +} + +//Logger struct for the Logger interface +type Logger interface { + EnableDebugMode() + EnableWriteToSTd() + StartSnapshot(d time.Duration) + GetLogFilePath() string + Info(s string) + Warning(s string) + Debug(s string) + Error(s string) +} + func NewLoggerFactory(logFilePath string) (Logger, error) { log, err := syslog.New(syslog.LOG_NOTICE, DefaultLbdTag) if err != nil { @@ -29,29 +56,35 @@ func NewLoggerFactory(logFilePath string) (Logger, error) { if err != nil { return nil, err } + basePath, extension, err := getLogFilePathAndExtension(logFilePath) + if err != nil { + return nil, fmt.Errorf("error while validating log file path. error: %v", err) + } + + if !isLogFilePathValid(basePath, extension) { + return nil, fmt.Errorf("invalid log file path. log path: %s", logFilePath) + } return &Log{ - logWriter: log, - filePath: logFilePath}, + logWriter: log, + logStartTime: time.Now(), + filePath: logFilePath, + logFileBasePath: basePath, + logFileExtension: extension, + }, nil } -//Log struct for the log -type Log struct { - logWriter *syslog.Writer - shouldWriteToSTD bool - isDebugAllowed bool - filePath string - logMu sync.Mutex +func isLogFilePathValid(basePath, extension string) bool { + return !strings.EqualFold(basePath, "") && !strings.EqualFold(extension, "") } -//Logger struct for the Logger interface -type Logger interface { - EnableDebugMode() - EnableWriteToSTd() - Info(s string) - Warning(s string) - Debug(s string) - Error(s string) +func getLogFilePathAndExtension(logFilePath string) (string, string, error) { + matcher := regexp.MustCompile(`(.*)\.(.*)`) + matchGroups := matcher.FindAllStringSubmatch(logFilePath, -1) + if len(matchGroups) == 0 || len(matchGroups[0]) < 3 { + return "", "", fmt.Errorf("log file path is not in the right format. path: %s", logFilePath) + } + return matchGroups[0][1], matchGroups[0][2], nil } func (l *Log) EnableDebugMode() { @@ -62,6 +95,30 @@ func (l *Log) EnableWriteToSTd() { l.shouldWriteToSTD = true } +func (l *Log) StartSnapshot(d time.Duration) { + if !l.isSnapShotEnabled { + l.isSnapShotEnabled = true + l.snapShotCycleTime = d + l.startSnapShot() + } +} + +func (l *Log) GetLogFilePath() string { + return l.filePath +} +func (l *Log) startSnapShot() { + l.logStartTime = time.Now() + l.filePath = fmt.Sprintf("%s_%s.%s", l.logFileBasePath, l.getFileNameTimeSuffix(), l.logFileExtension) +} + +func (l *Log) getFileNameTimeSuffix() string { + return fmt.Sprintf("%v_%v_%v-%v_%v_%v", l.logStartTime.Year(), l.logStartTime.Month(), l.logStartTime.Day(), l.logStartTime.Hour(), l.logStartTime.Minute(), l.logStartTime.Second()) +} + +func (l *Log) shouldStartNewSnapshot() bool { + return time.Now().Sub(l.logStartTime) >= l.snapShotCycleTime +} + //Info write as Info func (l *Log) Info(s string) { if l.logWriter != nil { @@ -114,12 +171,15 @@ func (l *Log) write(s string) { msg := fmt.Sprintf("%s %s[%d]: %s%s", timestamp, tag, os.Getpid(), s, nl) - l.logMu.Lock() - defer l.logMu.Unlock() if l.shouldWriteToSTD { fmt.Printf(msg) } + l.logMu.Lock() + defer l.logMu.Unlock() + if l.shouldStartNewSnapshot() { + l.startSnapShot() + } f, err := os.OpenFile(l.filePath, os.O_APPEND|os.O_WRONLY|os.O_CREATE, 0640) if err != nil { _, _ = fmt.Fprintf(os.Stderr, "error while opening the log file. error: %v", err) diff --git a/tests/lbcluster_log_test.go b/tests/lbcluster_log_test.go index d343b18..98003cd 100644 --- a/tests/lbcluster_log_test.go +++ b/tests/lbcluster_log_test.go @@ -1,120 +1,178 @@ package main_test import ( + "fmt" "io/ioutil" "lb-experts/golbd/lbcluster" + "log" "os" "strings" "testing" + "time" ) -const testLogFilePath = "../tests/sample.log" +const ( + testLogDirPath = "../tests" + testFileName = "sample" +) -func TestLBClusterLoggerForInitFailure(t *testing.T) { +func getLogFilePath() string { + return fmt.Sprintf("%s/%s.%s", testLogDirPath, testFileName, "log") +} +func TestLBClusterLoggerForInitFailure(t *testing.T) { _, err := lbcluster.NewLoggerFactory("") - if err==nil { + if err == nil { t.Errorf("expected error not thrown") } } -func TestLBClusterLoggerForInitSuccess(t *testing.T) { - defer deleteFile(t, testLogFilePath) - logger, err := lbcluster.NewLoggerFactory(testLogFilePath) - if err!=nil { +func TestLBClusterLoggerForInitSuccess(t *testing.T) { + defer deleteFile(t) + logger, err := lbcluster.NewLoggerFactory(getLogFilePath()) + if err != nil { t.Fail() - t.Errorf("unexpected error thrown. error: %v",err) + t.Errorf("unexpected error thrown. error: %v", err) } - if logger==nil{ + if logger == nil { t.Fail() t.Errorf("logger instance is nil") } } -func TestLBClusterLoggerForDebugDisabled(t *testing.T) { - defer deleteFile(t, testLogFilePath) - logger, err := lbcluster.NewLoggerFactory(testLogFilePath) - if err!=nil { +func TestLBClusterLoggerForSnapshot(t *testing.T) { + defer deleteFile(t) + logger, err := lbcluster.NewLoggerFactory(getLogFilePath()) + if err != nil { + t.Fail() + t.Errorf("unexpected error thrown. error: %v", err) + } + if logger == nil { + t.Fail() + t.Errorf("logger instance is nil") + } + logger.StartSnapshot(1 * time.Minute) + if !strings.Contains(logger.GetLogFilePath(), + fmt.Sprintf("%v_%v_%v-%v_%v", time.Now().Year(), time.Now().Month(), time.Now().Day(), time.Now().Hour(), time.Now().Minute())) { t.Fail() - t.Errorf("unexpected error thrown. error: %v",err) + t.Errorf("error while setting snapshot") + } +} + +func TestLBClusterLoggerForNewSnapshot(t *testing.T) { + defer deleteFile(t) + logger, err := lbcluster.NewLoggerFactory(getLogFilePath()) + if err != nil { + t.Fail() + t.Errorf("unexpected error thrown. error: %v", err) + } + if logger == nil { + t.Fail() + t.Errorf("logger instance is nil") + } + curTime := time.Now() + logger.StartSnapshot(5 * time.Second) + time.Sleep(5 * time.Second) + curTime = curTime.Add(5 * time.Second) + logger.Info("sample info") + if !strings.Contains(logger.GetLogFilePath(), + fmt.Sprintf("%v_%v_%v-%v_%v_%v", curTime.Year(), curTime.Month(), curTime.Day(), curTime.Hour(), curTime.Minute(), curTime.Second())) { + t.Fail() + t.Errorf("error while setting snapshot") + } +} + +func TestLBClusterLoggerForDebugDisabled(t *testing.T) { + defer deleteFile(t) + logger, err := lbcluster.NewLoggerFactory(getLogFilePath()) + if err != nil { + t.Fail() + t.Errorf("unexpected error thrown. error: %v", err) } logger.Debug("sample info") - if isLogPresentInFile(t, testLogFilePath, "sample info") { + if isLogPresentInFile(t, getLogFilePath(), "sample info") { t.Fail() t.Errorf("log file does not contain the expected debug info. Expected Info: %s", "sample info") } } -func TestLBClusterLoggerForDebugEnabled(t *testing.T) { - defer deleteFile(t, testLogFilePath) - logger, err := lbcluster.NewLoggerFactory(testLogFilePath) - if err!=nil { +func TestLBClusterLoggerForDebugEnabled(t *testing.T) { + defer deleteFile(t) + logger, err := lbcluster.NewLoggerFactory(getLogFilePath()) + if err != nil { t.Fail() - t.Errorf("unexpected error thrown. error: %v",err) + t.Errorf("unexpected error thrown. error: %v", err) } logger.EnableDebugMode() logger.Debug("sample info") - if !isLogPresentInFile(t, testLogFilePath, "sample info") { + if !isLogPresentInFile(t, getLogFilePath(), "sample info") { t.Fail() t.Errorf("log file does not contain the expected debug info. Expected Info: %s", "sample info") } } -func TestLBClusterLoggerForInfo(t *testing.T) { - defer deleteFile(t, testLogFilePath) - logger, err := lbcluster.NewLoggerFactory(testLogFilePath) - if err!=nil { +func TestLBClusterLoggerForInfo(t *testing.T) { + defer deleteFile(t) + logger, err := lbcluster.NewLoggerFactory(getLogFilePath()) + if err != nil { t.Fail() - t.Errorf("unexpected error thrown. error: %v",err) + t.Errorf("unexpected error thrown. error: %v", err) } logger.Info("sample info") - if !isLogPresentInFile(t, testLogFilePath, "INFO: sample info") { + if !isLogPresentInFile(t, getLogFilePath(), "INFO: sample info") { t.Fail() t.Errorf("log file does not contain the expected debug info. Expected Info: %s", "INFO: sample info") } } -func TestLBClusterLoggerForWarning(t *testing.T) { - defer deleteFile(t, testLogFilePath) - logger, err := lbcluster.NewLoggerFactory(testLogFilePath) - if err!=nil { +func TestLBClusterLoggerForWarning(t *testing.T) { + defer deleteFile(t) + logger, err := lbcluster.NewLoggerFactory(getLogFilePath()) + if err != nil { t.Fail() - t.Errorf("unexpected error thrown. error: %v",err) + t.Errorf("unexpected error thrown. error: %v", err) } logger.Warning("sample info") - if !isLogPresentInFile(t, testLogFilePath, "WARNING: sample info") { + if !isLogPresentInFile(t, getLogFilePath(), "WARNING: sample info") { t.Fail() t.Errorf("log file does not contain the expected debug info. Expected Info: %s", "WARNING: sample info") } } -func TestLBClusterLoggerForError(t *testing.T) { - defer deleteFile(t, testLogFilePath) - logger, err := lbcluster.NewLoggerFactory(testLogFilePath) - if err!=nil { +func TestLBClusterLoggerForError(t *testing.T) { + defer deleteFile(t) + logger, err := lbcluster.NewLoggerFactory(getLogFilePath()) + if err != nil { t.Fail() - t.Errorf("unexpected error thrown. error: %v",err) + t.Errorf("unexpected error thrown. error: %v", err) } logger.Error("sample info") - if !isLogPresentInFile(t, testLogFilePath, "ERROR: sample info") { + if !isLogPresentInFile(t, getLogFilePath(), "ERROR: sample info") { t.Fail() t.Errorf("log file does not contain the expected debug info. Expected Info: %s", "ERROR: sample info") } } -func deleteFile(t *testing.T, filePath string){ - err := os.Remove(filePath) - if err!=nil{ - t.Errorf("error whil deleting log file error: %v", err) +func deleteFile(t *testing.T) { + files, err := ioutil.ReadDir(testLogDirPath) + if err != nil { + log.Fatal(err) + } + + for _, f := range files { + if strings.HasPrefix(f.Name(), testFileName) { + err = os.Remove(f.Name()) + if err != nil { + t.Errorf("error whil deleting log file error: %v", err) + } + } } } -func isLogPresentInFile(t *testing.T, filePath string, stringToCheck string) bool{ +func isLogPresentInFile(t *testing.T, filePath string, stringToCheck string) bool { data, err := ioutil.ReadFile(filePath) - if err!=nil { + if err != nil { t.Errorf("error while reading log file. error: %v", err) return false } - return strings.Contains(string(data),stringToCheck) + return strings.Contains(string(data), stringToCheck) } - - From 883a2797b57bd2e4f1f3e53f606b6dc80b70fa3b Mon Sep 17 00:00:00 2001 From: Yadu Nandan Date: Wed, 11 May 2022 02:36:40 -0700 Subject: [PATCH 08/20] minor fixes --- lbconfig/config.go | 4 ++-- lbd.go | 39 +++++++++++++++++++++------------------ 2 files changed, 23 insertions(+), 20 deletions(-) diff --git a/lbconfig/config.go b/lbconfig/config.go index 5d7b466..534ebc0 100644 --- a/lbconfig/config.go +++ b/lbconfig/config.go @@ -45,7 +45,7 @@ type LBConfig struct { SnmpPassword string DNSManager string configFilePath string - lbLog *lbcluster.Log + lbLog lbcluster.Logger Clusters map[string][]string Parameters map[string]lbcluster.Params } @@ -60,7 +60,7 @@ func (fs ConfigFileChangeSignal) IsErrorPresent() bool { } // NewLoadBalancerConfig - instantiates a new load balancer config -func NewLoadBalancerConfig(configFilePath string, lbClusterLog *lbcluster.Log) Config { +func NewLoadBalancerConfig(configFilePath string, lbClusterLog lbcluster.Logger) Config { return &LBConfig{ configFilePath: configFilePath, lbLog: lbClusterLog, diff --git a/lbd.go b/lbd.go index 16a092b..1080765 100644 --- a/lbd.go +++ b/lbd.go @@ -114,7 +114,7 @@ func updateHeartbeat(config lbconfig.Config, hostname string, lg lbcluster.Logge return nil } -func updateHeartBeatToFile(heartBeatFilePath string, hostname string, lg *lbcluster.Log) error { +func updateHeartBeatToFile(heartBeatFilePath string, hostname string, lg lbcluster.Logger) error { secs := time.Now().Unix() f, err := os.OpenFile(heartBeatFilePath, os.O_APPEND|os.O_WRONLY|os.O_CREATE, 0644) defer f.Close() @@ -151,31 +151,34 @@ func sleep(seconds time.Duration, controlChan <-chan bool, waitGroup *sync.WaitG func main() { wg := sync.WaitGroup{} - log, e := syslog.New(syslog.LOG_NOTICE, DefaultLbdTag) - lg := lbcluster.Log{SyslogWriter: log, Stdout: *stdoutFlag, Debugflag: *debugFlag, TofilePath: *logFileFlag} + logger, err := lbcluster.NewLoggerFactory(*logFileFlag) + if err != nil { + fmt.Printf("error during log initialization. error: %v", err) + os.Exit(1) + } + + if *stdoutFlag { + logger.EnableWriteToSTd() + } controlChan := make(chan bool) defer close(controlChan) defer wg.Done() - defer lg.Error("The lbd is not supposed to stop") + defer logger.Error("The lbd is not supposed to stop") flag.Parse() if *versionFlag { fmt.Printf("This is a proof of concept golbd version: %s-%s \n", Version, Release) os.Exit(0) } rand.Seed(time.Now().UTC().UnixNano()) - if e != nil { - fmt.Printf("Error getting a syslog instance %v\nThe service will only write to the logfile %v\n\n", e, *logFileFlag) - } - - lg.Info("Starting lbd") - lbConfig := lbconfig.NewLoadBalancerConfig(*configFileFlag, &lg) + logger.Info("Starting lbd") + lbConfig := lbconfig.NewLoadBalancerConfig(*configFileFlag, logger) config, lbclusters, err := lbConfig.Load() if err != nil { - lg.Warning("loadConfig Error: ") - lg.Warning(err.Error()) + logger.Warning("loadConfig Error: ") + logger.Warning(err.Error()) os.Exit(1) } - lg.Info("Clusters loaded") + logger.Info("Clusters loaded") fileChangeSignal := lbConfig.WatchFileChange(controlChan, &wg) intervalTickerSignal := sleep(DefaultSleepDuration, controlChan, &wg) @@ -187,13 +190,13 @@ func main() { controlChan <- true return } - lg.Info("Config Changed") + logger.Info("Config Changed") config, lbclusters, err = lbConfig.Load() if err != nil { - lg.Error(fmt.Sprintf("Error getting the clusters (something wrong in %v", configFileFlag)) + logger.Error(fmt.Sprintf("Error getting the clusters (something wrong in %v", configFileFlag)) } case <-intervalTickerSignal: - checkAliases(config, lg, lbclusters) + checkAliases(config, logger, lbclusters) break } } @@ -239,7 +242,7 @@ func checkAliases(config lbconfig.Config, lg lbcluster.Logger, lbclusters []lbcl lg.Debug("All the hosts have been tested") - updateDNS = shouldUpdateDNS(config, hostname, &lg) + updateDNS = shouldUpdateDNS(config, hostname, lg) /* Finally, let's go through the aliases, selecting the best hosts*/ for _, pc := range clustersToUpdate { @@ -263,7 +266,7 @@ func checkAliases(config lbconfig.Config, lg lbcluster.Logger, lbclusters []lbcl } if updateDNS { - updateHeartbeat(config, hostname, &lg) + updateHeartbeat(config, hostname, lg) } lg.Debug("iteration done!") From c848bfd522ea85356f2a37a1b5f4b855d53d4173 Mon Sep 17 00:00:00 2001 From: Yadu Nandan Date: Sun, 8 May 2022 17:53:28 -0700 Subject: [PATCH 09/20] added concurrency to host evauluation --- lbcluster/lbcluster.go | 37 +++++++++++++++++++++++++------------ 1 file changed, 25 insertions(+), 12 deletions(-) diff --git a/lbcluster/lbcluster.go b/lbcluster/lbcluster.go index 123de8a..14e480c 100644 --- a/lbcluster/lbcluster.go +++ b/lbcluster/lbcluster.go @@ -7,6 +7,7 @@ import ( "math/rand" "net" "net/http" + "sync" "sort" "strings" "time" @@ -80,8 +81,9 @@ func Shuffle(n int, swap func(i, j int)) error { //Node Struct to keep the ips and load of a node for an alias type Node struct { - Load int - IPs []net.IP + Load int + IPs []net.IP + HostName string } //NodeList struct for the list @@ -297,17 +299,28 @@ func (lbc *LBCluster) checkRogerState(host string) string { //EvaluateHosts gets the load from the all the nodes func (lbc *LBCluster) EvaluateHosts(hostsToCheck map[string]lbhost.Host) { - - for currenthost := range lbc.Host_metric_table { - host := hostsToCheck[currenthost] - ips, err := host.GetWorkingIPs() - if err != nil { - ips, err = host.GetIps() - } - - lbc.Host_metric_table[currenthost] = Node{host.GetLoadForAlias(lbc.ClusterConfig.Cluster_name), ips} - lbc.Slog.Debug(fmt.Sprintf("node: %s It has a load of %d", currenthost, lbc.Host_metric_table[currenthost].Load)) + var nodeChan = make(chan Node) + defer close(nodeChan) + var wg sync.WaitGroup + for currentHost := range lbc.Host_metric_table { + wg.Add(1) + go func(selectedHost string) { + host := hostsToCheck[selectedHost] + ips, err := host.GetWorkingIPs() + if err != nil { + ips, err = host.GetIps() + } + nodeChan <- Node{host.GetLoadForAlias(lbc.ClusterConfig.Cluster_name), ips, selectedHost} + }(currentHost) } + go func() { + for nodeData := range nodeChan { + wg.Done() + lbc.Host_metric_table[nodeData.HostName] = nodeData + lbc.Slog.Debug(fmt.Sprintf("node: %s It has a load of %d", nodeData.HostName, lbc.Host_metric_table[nodeData.HostName].Load)) + } + }() + wg.Wait() } //ReEvaluateHostsForMinimum gets the load from the all the nodes for Minimum metric policy From f98b278dcdb1576c6fcbdc47dcd00b824849c7e9 Mon Sep 17 00:00:00 2001 From: Yadu Nandan Date: Thu, 12 May 2022 01:44:45 -0700 Subject: [PATCH 10/20] test refractor --- lbcluster/lbcluster.go | 18 +- lbconfig/config.go | 60 +++++- lbd.go | 14 +- lbhost/lbhost.go | 28 ++- {lbcluster => logger}/lbcluster_log.go | 2 +- model/cluster_config.go | 7 + tests/aliasload_test.go | 41 ++-- tests/evaluate_hosts_internal_test.go | 73 ++++++- tests/find_best_hosts_test.go | 28 ++- tests/get_list_hosts_test.go | 144 +++++++------- tests/get_state_dns_test.go | 11 +- tests/lbcluster_log_test.go | 20 +- tests/loadClusters_test.go | 255 ++++++++++++------------- tests/loadConfig_test.go | 26 +-- tests/sample.log | 0 15 files changed, 425 insertions(+), 302 deletions(-) rename {lbcluster => logger}/lbcluster_log.go (99%) create mode 100644 model/cluster_config.go create mode 100644 tests/sample.log diff --git a/lbcluster/lbcluster.go b/lbcluster/lbcluster.go index 14e480c..a05d5ed 100644 --- a/lbcluster/lbcluster.go +++ b/lbcluster/lbcluster.go @@ -4,12 +4,14 @@ import ( "encoding/json" "fmt" "io/ioutil" + "lb-experts/golbd/logger" + "lb-experts/golbd/model" "math/rand" "net" "net/http" - "sync" "sort" "strings" + "sync" "time" "lb-experts/golbd/lbhost" @@ -26,20 +28,14 @@ const OID string = ".1.3.6.1.4.1.96.255.1" //LBCluster struct of an lbcluster alias type LBCluster struct { - ClusterConfig Config + ClusterConfig model.CluserConfig Host_metric_table map[string]Node Parameters Params Time_of_last_evaluation time.Time Current_best_ips []net.IP Previous_best_ips_dns []net.IP Current_index int - Slog Logger -} - -type Config struct { - Cluster_name string - Loadbalancing_username string - Loadbalancing_password string + Slog logger.Logger } //Params of the alias @@ -315,9 +311,9 @@ func (lbc *LBCluster) EvaluateHosts(hostsToCheck map[string]lbhost.Host) { } go func() { for nodeData := range nodeChan { - wg.Done() lbc.Host_metric_table[nodeData.HostName] = nodeData lbc.Slog.Debug(fmt.Sprintf("node: %s It has a load of %d", nodeData.HostName, lbc.Host_metric_table[nodeData.HostName].Load)) + wg.Done() } }() wg.Wait() @@ -332,7 +328,7 @@ func (lbc *LBCluster) ReEvaluateHostsForMinimum(hostsToCheck map[string]lbhost.H if err != nil { ips, err = host.GetIps() } - lbc.Host_metric_table[currenthost] = Node{host.GetLoadForAlias(lbc.ClusterConfig.Cluster_name), ips} + lbc.Host_metric_table[currenthost] = Node{host.GetLoadForAlias(lbc.ClusterConfig.Cluster_name), ips, host.GetName()} lbc.Slog.Debug(fmt.Sprintf("node: %s It has a load of %d", currenthost, lbc.Host_metric_table[currenthost].Load)) } } diff --git a/lbconfig/config.go b/lbconfig/config.go index 534ebc0..dbb8001 100644 --- a/lbconfig/config.go +++ b/lbconfig/config.go @@ -5,6 +5,8 @@ import ( "encoding/json" "fmt" "io" + "lb-experts/golbd/logger" + "lb-experts/golbd/model" "net" "os" "strconv" @@ -31,6 +33,18 @@ type Config interface { UnlockHeartBeatMutex() WatchFileChange(controlChan <-chan bool, waitGroup *sync.WaitGroup) <-chan ConfigFileChangeSignal Load() (*LBConfig, []lbcluster.LBCluster, error) + + // testing only + SetMasterHost(masterHostName string) + SetHeartBeatFileName(heartBeatFileName string) + SetHeartBeatDirPath(heartBeatDirPath string) + SetDNSManager(dnsManager string) + SetTSIGKeyPrefix(tsigKeyPrefix string) + SetTSIGInternalKey(tsigInternalKey string) + SetTSIGExternalKey(tsigExternalKey string) + SetClusters(clusters map[string][]string) + SetSNMPPassword(password string) + SetParameters(params map[string]lbcluster.Params) } // Config this is the configuration of the lbd @@ -45,7 +59,7 @@ type LBConfig struct { SnmpPassword string DNSManager string configFilePath string - lbLog lbcluster.Logger + lbLog logger.Logger Clusters map[string][]string Parameters map[string]lbcluster.Params } @@ -60,7 +74,7 @@ func (fs ConfigFileChangeSignal) IsErrorPresent() bool { } // NewLoadBalancerConfig - instantiates a new load balancer config -func NewLoadBalancerConfig(configFilePath string, lbClusterLog lbcluster.Logger) Config { +func NewLoadBalancerConfig(configFilePath string, lbClusterLog logger.Logger) Config { return &LBConfig{ configFilePath: configFilePath, lbLog: lbClusterLog, @@ -71,30 +85,70 @@ func (c *LBConfig) GetMasterHost() string { return c.Master } +func (c *LBConfig) SetMasterHost(masterHostName string) { + c.Master = masterHostName +} + func (c *LBConfig) GetHeartBeatFileName() string { return c.HeartbeatFile } +func (c *LBConfig) SetHeartBeatFileName(heartBeatFileName string) { + c.HeartbeatFile = heartBeatFileName +} + func (c *LBConfig) GetHeartBeatDirPath() string { return c.HeartbeatPath } +func (c *LBConfig) SetHeartBeatDirPath(heartBeatDirPath string) { + c.HeartbeatPath = heartBeatDirPath +} + func (c *LBConfig) GetDNSManager() string { return c.DNSManager } +func (c *LBConfig) SetDNSManager(dnsManager string) { + c.DNSManager = dnsManager +} + func (c *LBConfig) GetTSIGKeyPrefix() string { return c.TsigKeyPrefix } +func (c *LBConfig) SetTSIGKeyPrefix(tsigKeyPrefix string) { + c.TsigKeyPrefix = tsigKeyPrefix +} + func (c *LBConfig) GetTSIGInternalKey() string { return c.TsigInternalKey } +func (c *LBConfig) SetTSIGInternalKey(tsigInternalKey string) { + c.TsigInternalKey = tsigInternalKey +} + func (c *LBConfig) GetTSIGExternalKey() string { return c.TsigExternalKey } +func (c *LBConfig) SetTSIGExternalKey(tsigExternalKey string) { + c.TsigExternalKey = tsigExternalKey +} + +func (c *LBConfig) SetClusters(clusters map[string][]string) { + c.Clusters = clusters +} + +func (c *LBConfig) SetParameters(params map[string]lbcluster.Params) { + c.Parameters = params +} + +func (c *LBConfig) SetSNMPPassword(password string) { + c.SnmpPassword = password +} + func (c *LBConfig) LockHeartBeatMutex() { c.HeartbeatMu.Lock() } @@ -247,7 +301,7 @@ func (c *LBConfig) loadClusters() ([]lbcluster.LBCluster, error) { continue } if par, ok := c.Parameters[k]; ok { - lbcConfig := lbcluster.Config{ + lbcConfig := model.CluserConfig{ Cluster_name: k, Loadbalancing_username: DefaultLoadBalancerConfig, Loadbalancing_password: c.SnmpPassword, diff --git a/lbd.go b/lbd.go index 1080765..b31aeb8 100644 --- a/lbd.go +++ b/lbd.go @@ -6,8 +6,8 @@ import ( "io/ioutil" "lb-experts/golbd/lbcluster" "lb-experts/golbd/lbhost" + "lb-experts/golbd/logger" "log" - "log/syslog" "math/rand" "os" "regexp" @@ -50,7 +50,7 @@ type ConfigFileChangeSignal struct { readError error } -func shouldUpdateDNS(config lbconfig.Config, hostname string, lg lbcluster.Logger) bool { +func shouldUpdateDNS(config lbconfig.Config, hostname string, lg logger.Logger) bool { if strings.EqualFold(hostname, config.GetMasterHost()) { return true } @@ -91,7 +91,7 @@ func shouldUpdateDNS(config lbconfig.Config, hostname string, lg lbcluster.Logge } -func updateHeartbeat(config lbconfig.Config, hostname string, lg lbcluster.Logger) error { +func updateHeartbeat(config lbconfig.Config, hostname string, lg logger.Logger) error { if hostname != config.GetMasterHost() { return nil } @@ -114,7 +114,7 @@ func updateHeartbeat(config lbconfig.Config, hostname string, lg lbcluster.Logge return nil } -func updateHeartBeatToFile(heartBeatFilePath string, hostname string, lg lbcluster.Logger) error { +func updateHeartBeatToFile(heartBeatFilePath string, hostname string, lg logger.Logger) error { secs := time.Now().Unix() f, err := os.OpenFile(heartBeatFilePath, os.O_APPEND|os.O_WRONLY|os.O_CREATE, 0644) defer f.Close() @@ -151,7 +151,7 @@ func sleep(seconds time.Duration, controlChan <-chan bool, waitGroup *sync.WaitG func main() { wg := sync.WaitGroup{} - logger, err := lbcluster.NewLoggerFactory(*logFileFlag) + logger, err := logger.NewLoggerFactory(*logFileFlag) if err != nil { fmt.Printf("error during log initialization. error: %v", err) os.Exit(1) @@ -190,7 +190,7 @@ func main() { controlChan <- true return } - logger.Info("Config Changed") + logger.Info("CluserConfig Changed") config, lbclusters, err = lbConfig.Load() if err != nil { logger.Error(fmt.Sprintf("Error getting the clusters (something wrong in %v", configFileFlag)) @@ -203,7 +203,7 @@ func main() { } // todo: add some tests -func checkAliases(config lbconfig.Config, lg lbcluster.Logger, lbclusters []lbcluster.LBCluster) { +func checkAliases(config lbconfig.Config, lg logger.Logger, lbclusters []lbcluster.LBCluster) { hostCheckChannel := make(chan lbhost.Host) defer close(hostCheckChannel) diff --git a/lbhost/lbhost.go b/lbhost/lbhost.go index 0c824f2..97ade1b 100644 --- a/lbhost/lbhost.go +++ b/lbhost/lbhost.go @@ -2,13 +2,14 @@ package lbhost import ( "fmt" + "lb-experts/golbd/logger" + "lb-experts/golbd/model" "net" "regexp" "strconv" "time" "github.com/reguero/go-snmplib" - "lb-experts/golbd/lbcluster" ) const ( @@ -25,36 +26,51 @@ type LBHostTransportResult struct { Response_error string } type LBHost struct { - ClusterConfig lbcluster.Config + ClusterConfig model.CluserConfig Host_name string HostTransports []LBHostTransportResult - Logger lbcluster.Logger + Logger logger.Logger } type Host interface { GetName() string + SetName(name string) SNMPDiscovery() - GetClusterConfig() *lbcluster.Config + GetClusterConfig() *model.CluserConfig GetLoadForAlias(clusterName string) int GetWorkingIPs() ([]net.IP, error) GetAllIPs() ([]net.IP, error) GetIps() ([]net.IP, error) + SetTransportPayload(transportPayloadList []LBHostTransportResult) + GetHostTransportPayloads() []LBHostTransportResult } -func NewLBHost(clusterConfig lbcluster.Config, logger lbcluster.Logger) Host { +func NewLBHost(clusterConfig model.CluserConfig, logger logger.Logger) Host { return &LBHost{ ClusterConfig: clusterConfig, Logger: logger, } } +func (lh *LBHost) SetName(name string) { + lh.Host_name = name +} + func (lh *LBHost) GetName() string { return lh.Host_name } -func (lh *LBHost) GetClusterConfig() *lbcluster.Config { +func (lh *LBHost) GetClusterConfig() *model.CluserConfig { return &lh.ClusterConfig } +func (lh *LBHost) GetHostTransportPayloads() []LBHostTransportResult { + return lh.HostTransports +} + +func (lh *LBHost) SetTransportPayload(transportPayloadList []LBHostTransportResult) { + lh.HostTransports = transportPayloadList +} + // todo: refractor into smaller functions func (lh *LBHost) SNMPDiscovery() { lh.find_transports() diff --git a/lbcluster/lbcluster_log.go b/logger/lbcluster_log.go similarity index 99% rename from lbcluster/lbcluster_log.go rename to logger/lbcluster_log.go index 3ee4732..584509d 100644 --- a/lbcluster/lbcluster_log.go +++ b/logger/lbcluster_log.go @@ -1,4 +1,4 @@ -package lbcluster +package logger import ( "fmt" diff --git a/model/cluster_config.go b/model/cluster_config.go new file mode 100644 index 0000000..14038bb --- /dev/null +++ b/model/cluster_config.go @@ -0,0 +1,7 @@ +package model + +type CluserConfig struct { + Cluster_name string + Loadbalancing_username string + Loadbalancing_password string +} diff --git a/tests/aliasload_test.go b/tests/aliasload_test.go index c3d540d..9bac85c 100644 --- a/tests/aliasload_test.go +++ b/tests/aliasload_test.go @@ -1,49 +1,48 @@ package main_test import ( + "lb-experts/golbd/lbhost" "reflect" "testing" - - "gitlab.cern.ch/lb-experts/golbd/lbhost" ) //Function TestGetLoadHosts tests the function GetLoadForAlias func TestGetLoadHosts(t *testing.T) { - hosts := []lbhost.LBHost{ + hosts := []lbhost.Host{ getHost("lxplus132.cern.ch", 7, ""), getHost("lxplus132.cern.ch", 0, "blabla.cern.ch=179,blablabla2.cern.ch=4"), getHost("toto132.lxplus.cern.ch", 42, ""), getHost("toto132.lxplus.cern.ch", 0, "blabla.subdo.cern.ch=179,blablabla2.subdo.cern.ch=4"), } - expectedhost0 := hosts[0].Host_transports[0].Response_int + expectedhost0 := hosts[0].GetHostTransportPayloads()[0].Response_int //expectedhost1 := hosts[1].HostTransports[0].Response_int - expectedhost2 := hosts[2].Host_transports[0].Response_int + expectedhost2 := hosts[2].GetHostTransportPayloads()[0].Response_int //expectedhost3 := hosts[3].HostTransports[0].Response_int - if !reflect.DeepEqual(hosts[0].Get_load_for_alias(hosts[0].Cluster_name), expectedhost0) { - t.Errorf(" got\n%v\nexpected\n%v", hosts[0].Get_load_for_alias(hosts[0].Cluster_name), expectedhost0) + if !reflect.DeepEqual(hosts[0].GetLoadForAlias(hosts[0].GetClusterConfig().Cluster_name), expectedhost0) { + t.Errorf(" got\n%v\nexpected\n%v", hosts[0].GetLoadForAlias(hosts[0].GetClusterConfig().Cluster_name), expectedhost0) } - if !reflect.DeepEqual(hosts[1].Get_load_for_alias(hosts[1].Cluster_name), 0) { - t.Errorf(" got\n%v\nexpected\n%v", hosts[1].Get_load_for_alias(hosts[1].Cluster_name), 0) + if !reflect.DeepEqual(hosts[1].GetLoadForAlias(hosts[1].GetClusterConfig().Cluster_name), 0) { + t.Errorf(" got\n%v\nexpected\n%v", hosts[1].GetLoadForAlias(hosts[1].GetClusterConfig().Cluster_name), 0) } - if !reflect.DeepEqual(hosts[1].Get_load_for_alias("blabla.cern.ch"), 179) { - t.Errorf(" got\n%v\nexpected\n%v", hosts[1].Get_load_for_alias("blabla.cern.ch"), 179) + if !reflect.DeepEqual(hosts[1].GetLoadForAlias("blabla.cern.ch"), 179) { + t.Errorf(" got\n%v\nexpected\n%v", hosts[1].GetLoadForAlias("blabla.cern.ch"), 179) } - if !reflect.DeepEqual(hosts[1].Get_load_for_alias("blablabla2.cern.ch"), 4) { - t.Errorf(" got\n%v\nexpected\n%v", hosts[1].Get_load_for_alias("blablabla2.cern.ch"), 4) + if !reflect.DeepEqual(hosts[1].GetLoadForAlias("blablabla2.cern.ch"), 4) { + t.Errorf(" got\n%v\nexpected\n%v", hosts[1].GetLoadForAlias("blablabla2.cern.ch"), 4) } - if !reflect.DeepEqual(hosts[2].Get_load_for_alias(hosts[2].Cluster_name), expectedhost2) { - t.Errorf(" got\n%v\nexpected\n%v", hosts[2].Get_load_for_alias(hosts[2].Cluster_name), expectedhost2) + if !reflect.DeepEqual(hosts[2].GetLoadForAlias(hosts[2].GetClusterConfig().Cluster_name), expectedhost2) { + t.Errorf(" got\n%v\nexpected\n%v", hosts[2].GetLoadForAlias(hosts[2].GetClusterConfig().Cluster_name), expectedhost2) } - if !reflect.DeepEqual(hosts[2].Get_load_for_alias("toto.subdo.cern.ch"), expectedhost2) { - t.Errorf(" got\n%v\nexpected\n%v", hosts[2].Get_load_for_alias("toto.subdo.cern.ch"), expectedhost2) + if !reflect.DeepEqual(hosts[2].GetLoadForAlias("toto.subdo.cern.ch"), expectedhost2) { + t.Errorf(" got\n%v\nexpected\n%v", hosts[2].GetLoadForAlias("toto.subdo.cern.ch"), expectedhost2) } - if !reflect.DeepEqual(hosts[3].Get_load_for_alias("blabla.subdo.cern.ch"), 179) { - t.Errorf(" got\n%v\nexpected\n%v", hosts[3].Get_load_for_alias("blabla.subdo.cern.ch"), 179) + if !reflect.DeepEqual(hosts[3].GetLoadForAlias("blabla.subdo.cern.ch"), 179) { + t.Errorf(" got\n%v\nexpected\n%v", hosts[3].GetLoadForAlias("blabla.subdo.cern.ch"), 179) } - if !reflect.DeepEqual(hosts[3].Get_load_for_alias("blablabla2.subdo.cern.ch"), 4) { - t.Errorf(" got\n%v\nexpected\n%v", hosts[3].Get_load_for_alias("blablabla2.subdo.cern.ch"), 4) + if !reflect.DeepEqual(hosts[3].GetLoadForAlias("blablabla2.subdo.cern.ch"), 4) { + t.Errorf(" got\n%v\nexpected\n%v", hosts[3].GetLoadForAlias("blablabla2.subdo.cern.ch"), 4) } } diff --git a/tests/evaluate_hosts_internal_test.go b/tests/evaluate_hosts_internal_test.go index 4387b8e..9b3a9fb 100644 --- a/tests/evaluate_hosts_internal_test.go +++ b/tests/evaluate_hosts_internal_test.go @@ -1,13 +1,66 @@ package main_test import ( + "fmt" + "lb-experts/golbd/lbcluster" + "lb-experts/golbd/lbhost" + "lb-experts/golbd/logger" + "lb-experts/golbd/model" "net" + "os" "reflect" "testing" - - "gitlab.cern.ch/lb-experts/golbd/lbcluster" + "time" ) +type mockHost struct { +} + +func (m mockHost) GetHostTransportPayloads() []lbhost.LBHostTransportResult { + panic("implement me") +} + +func (m mockHost) SetName(name string) { + panic("implement me") +} + +func (m mockHost) SetTransportPayload(transportPayloadList []lbhost.LBHostTransportResult) { + panic("implement me") +} + +func (m mockHost) GetName() string { + panic("implement me") +} + +func (m mockHost) SNMPDiscovery() { + panic("implement me") +} + +func (m mockHost) GetClusterConfig() *model.CluserConfig { + panic("implement me") +} + +func (m mockHost) GetLoadForAlias(clusterName string) int { + return 0 +} + +func (m mockHost) GetWorkingIPs() ([]net.IP, error) { + return []net.IP{}, fmt.Errorf("sample error") +} + +func (m mockHost) GetAllIPs() ([]net.IP, error) { + panic("implement me") +} + +func (m mockHost) GetIps() ([]net.IP, error) { + time.Sleep(5 * time.Second) // simulating a network request + return []net.IP{}, nil +} + +func NewMockHost() lbhost.Host { + return &mockHost{} +} + func compareIPs(t *testing.T, source, target []net.IP) { found := map[string]bool{} @@ -68,3 +121,19 @@ func TestEvaluateHosts(t *testing.T) { t.Errorf("e.evaluate_hosts: c.Time_of_last_evaluation: got\n%v\nexpected\n%v", c.Time_of_last_evaluation, expectedTimeOfLastEvaluation) } } + +func TestEvaluateHostsConcurrency(t *testing.T) { + mockHostMap := make(map[string]lbhost.Host) + mockHostMap["sampleHost"] = NewMockHost() + logger, _ := logger.NewLoggerFactory("sample.log") + cluster := lbcluster.LBCluster{Slog: logger} + cluster.Host_metric_table = map[string]lbcluster.Node{"sampleHost": {HostName: "sampleHost"}} + startTime := time.Now() + cluster.EvaluateHosts(mockHostMap) + endTime := time.Now() + if endTime.Sub(startTime) > 6*time.Second { + t.Fail() + t.Errorf("concurrent job not running properly. expDuration:%v, actualDuration:%v", 6, endTime.Sub(startTime)) + } + os.Remove("sample.log") +} diff --git a/tests/find_best_hosts_test.go b/tests/find_best_hosts_test.go index 2ccc939..ea2cfd2 100644 --- a/tests/find_best_hosts_test.go +++ b/tests/find_best_hosts_test.go @@ -27,8 +27,11 @@ func TestFindBestHosts(t *testing.T) { expected_host_metric_table := getExpectedHostMetric() expected_current_best_ips := []net.IP{net.ParseIP("2001:1458:d00:2c::100:a6"), net.ParseIP("188.184.108.98"), net.ParseIP("2001:1458:d00:32::100:51"), net.ParseIP("188.184.116.81")} - - if !c.FindBestHosts(hosts_to_check) { + isDNSUpdateValid, err := c.FindBestHosts(hosts_to_check) + if err != nil { + t.Errorf("Error while finding best hosts. error:%v", err) + } + if !isDNSUpdateValid { t.Errorf("e.Find_best_hosts: returned false, expected true") } if !reflect.DeepEqual(c.Host_metric_table, expected_host_metric_table) { @@ -53,8 +56,11 @@ func TestFindBestHostsNoValidHostCmsfrontier(t *testing.T) { expected_current_best_ips := []net.IP{} expected_time_of_last_evaluation := c.Time_of_last_evaluation - - if c.FindBestHosts(bad_hosts_to_check) { + isDNSUpdateValid, err := c.FindBestHosts(bad_hosts_to_check) + if err != nil { + t.Errorf("Error while finding best hosts. error:%v", err) + } + if isDNSUpdateValid { t.Errorf("e.Find_best_hosts: returned true, expected false") } if !reflect.DeepEqual(c.Current_best_ips, expected_current_best_ips) { @@ -74,8 +80,11 @@ func TestFindBestHostsNoValidHostMinino(t *testing.T) { bad_hosts_to_check := getBadHostsToCheck(c) expected_current_best_ips := []net.IP{} - - if !c.FindBestHosts(bad_hosts_to_check) { + isDNSUpdateValid, err := c.FindBestHosts(bad_hosts_to_check) + if err != nil { + t.Errorf("Error while finding best hosts. error:%v", err) + } + if !isDNSUpdateValid { t.Errorf("e.Find_best_hosts: returned false, expected true") } if !reflect.DeepEqual(c.Current_best_ips, expected_current_best_ips) { @@ -95,8 +104,11 @@ func TestFindBestHostsNoValidHostMinimum(t *testing.T) { bad_hosts_to_check := getBadHostsToCheck(c) not_expected_current_best_ips := []net.IP{} - - if !c.FindBestHosts(bad_hosts_to_check) { + isDNSUpdateValid, err := c.FindBestHosts(bad_hosts_to_check) + if err != nil { + t.Errorf("Error while finding best hosts. error:%v", err) + } + if !isDNSUpdateValid { t.Errorf("e.Find_best_hosts: returned false, expected true") } if reflect.DeepEqual(c.Current_best_ips, not_expected_current_best_ips) { diff --git a/tests/get_list_hosts_test.go b/tests/get_list_hosts_test.go index eab3360..2d82a48 100644 --- a/tests/get_list_hosts_test.go +++ b/tests/get_list_hosts_test.go @@ -1,56 +1,36 @@ package main_test import ( + "lb-experts/golbd/lbcluster" + "lb-experts/golbd/lbhost" + "lb-experts/golbd/logger" + "lb-experts/golbd/model" "net" "reflect" "testing" - - "gitlab.cern.ch/lb-experts/golbd/lbcluster" - "gitlab.cern.ch/lb-experts/golbd/lbhost" ) func TestGetListHostsOne(t *testing.T) { c := getTestCluster("test01.cern.ch") - - expected := map[string]lbhost.LBHost{ - "lxplus041.cern.ch": lbhost.LBHost{Cluster_name: c.Cluster_name, - Host_name: "lxplus041.cern.ch", - Loadbalancing_username: c.Loadbalancing_username, - Loadbalancing_password: c.Loadbalancing_password, - LogFile: c.Slog.TofilePath, - Debugflag: c.Slog.Debugflag, - }, - "monit-kafkax-17be060b0d.cern.ch": lbhost.LBHost{Cluster_name: c.Cluster_name, - Host_name: "monit-kafkax-17be060b0d.cern.ch", - Loadbalancing_username: c.Loadbalancing_username, - Loadbalancing_password: c.Loadbalancing_password, - LogFile: c.Slog.TofilePath, - Debugflag: c.Slog.Debugflag, - }, - "lxplus132.cern.ch": lbhost.LBHost{Cluster_name: c.Cluster_name, - Host_name: "lxplus132.cern.ch", - Loadbalancing_username: c.Loadbalancing_username, - Loadbalancing_password: c.Loadbalancing_password, - LogFile: c.Slog.TofilePath, - Debugflag: c.Slog.Debugflag, - }, - "lxplus133.subdo.cern.ch": lbhost.LBHost{Cluster_name: c.Cluster_name, - Host_name: "lxplus133.subdo.cern.ch", - Loadbalancing_username: c.Loadbalancing_username, - Loadbalancing_password: c.Loadbalancing_password, - LogFile: c.Slog.TofilePath, - Debugflag: c.Slog.Debugflag, - }, - "lxplus130.cern.ch": lbhost.LBHost{Cluster_name: c.Cluster_name, - Host_name: "lxplus130.cern.ch", - Loadbalancing_username: c.Loadbalancing_username, - Loadbalancing_password: c.Loadbalancing_password, - LogFile: c.Slog.TofilePath, - Debugflag: c.Slog.Debugflag, - }, + host1 := lbhost.NewLBHost(c.ClusterConfig, c.Slog) + host1.SetName("lxplus041.cern.ch") + host2 := lbhost.NewLBHost(c.ClusterConfig, c.Slog) + host2.SetName("monit-kafkax-17be060b0d.cern.ch") + host3 := lbhost.NewLBHost(c.ClusterConfig, c.Slog) + host3.SetName("lxplus132.cern.ch") + host4 := lbhost.NewLBHost(c.ClusterConfig, c.Slog) + host4.SetName("lxplus041.cern.ch") + host5 := lbhost.NewLBHost(c.ClusterConfig, c.Slog) + host5.SetName("lxplus041.cern.ch") + expected := map[string]lbhost.Host{ + "lxplus041.cern.ch": host1, + "monit-kafkax-17be060b0d.cern.ch": host2, + "lxplus132.cern.ch": host3, + "lxplus133.subdo.cern.ch": host4, + "lxplus130.cern.ch": host5, } - hosts_to_check := make(map[string]lbhost.LBHost) + hosts_to_check := make(map[string]lbhost.Host) c.Get_list_hosts(hosts_to_check) if !reflect.DeepEqual(hosts_to_check, expected) { t.Errorf("e.Get_list_hosts: got\n%v\nexpected\n%v", hosts_to_check, expected) @@ -58,63 +38,67 @@ func TestGetListHostsOne(t *testing.T) { } func TestGetListHostsTwo(t *testing.T) { - lg := lbcluster.Log{Stdout: true, Debugflag: false} + logger, _ := logger.NewLoggerFactory("") + logger.EnableWriteToSTd() clusters := []lbcluster.LBCluster{ - {Cluster_name: "test01.cern.ch", + {ClusterConfig: model.CluserConfig{ + Cluster_name: "test01.cern.ch", Loadbalancing_username: "loadbalancing", Loadbalancing_password: "zzz123", - Host_metric_table: map[string]lbcluster.Node{"lxplus142.cern.ch": lbcluster.Node{}, "lxplus177.cern.ch": lbcluster.Node{}}, - Parameters: lbcluster.Params{Behaviour: "mindless", Best_hosts: 2, External: true, Metric: "cmsfrontier", Polling_interval: 6, Statistics: "long"}, + }, Host_metric_table: map[string]lbcluster.Node{"lxplus142.cern.ch": lbcluster.Node{}, "lxplus177.cern.ch": lbcluster.Node{}}, + Parameters: lbcluster.Params{Behaviour: "mindless", Best_hosts: 2, External: true, Metric: "cmsfrontier", Polling_interval: 6, Statistics: "long"}, //Time_of_last_evaluation time.Time Current_best_ips: []net.IP{}, Previous_best_ips_dns: []net.IP{}, - Slog: &lg, + Slog: logger, Current_index: 0}, - lbcluster.LBCluster{Cluster_name: "test02.cern.ch", + lbcluster.LBCluster{ClusterConfig: model.CluserConfig{ + Cluster_name: "test02.cern.ch", Loadbalancing_username: "loadbalancing", Loadbalancing_password: "zzz123", - Host_metric_table: map[string]lbcluster.Node{"lxplus013.cern.ch": lbcluster.Node{}, "lxplus177.cern.ch": lbcluster.Node{}, "lxplus025.cern.ch": lbcluster.Node{}}, - Parameters: lbcluster.Params{Behaviour: "mindless", Best_hosts: 10, External: false, Metric: "cmsfrontier", Polling_interval: 6, Statistics: "long"}, + }, + Host_metric_table: map[string]lbcluster.Node{"lxplus013.cern.ch": lbcluster.Node{}, "lxplus177.cern.ch": lbcluster.Node{}, "lxplus025.cern.ch": lbcluster.Node{}}, + Parameters: lbcluster.Params{Behaviour: "mindless", Best_hosts: 10, External: false, Metric: "cmsfrontier", Polling_interval: 6, Statistics: "long"}, //Time_of_last_evaluation time.Time Current_best_ips: []net.IP{}, Previous_best_ips_dns: []net.IP{}, - Slog: &lg, + Slog: logger, Current_index: 0}} - expected := map[string]lbhost.LBHost{ - "lxplus142.cern.ch": lbhost.LBHost{Cluster_name: "test01.cern.ch", - Host_name: "lxplus142.cern.ch", - Loadbalancing_username: "loadbalancing", - Loadbalancing_password: "zzz123", - LogFile: "", - Debugflag: false, - }, - "lxplus177.cern.ch": lbhost.LBHost{Cluster_name: "test01.cern.ch,test02.cern.ch", - Host_name: "lxplus177.cern.ch", - Loadbalancing_username: "loadbalancing", - Loadbalancing_password: "zzz123", - LogFile: "", - Debugflag: false, - }, - "lxplus013.cern.ch": lbhost.LBHost{Cluster_name: "test02.cern.ch", - Host_name: "lxplus013.cern.ch", - Loadbalancing_username: "loadbalancing", - Loadbalancing_password: "zzz123", - LogFile: "", - Debugflag: false, - }, - "lxplus025.cern.ch": lbhost.LBHost{Cluster_name: "test02.cern.ch", - Host_name: "lxplus025.cern.ch", - Loadbalancing_username: "loadbalancing", - Loadbalancing_password: "zzz123", - LogFile: "", - Debugflag: false, - }, + host1 := lbhost.NewLBHost(model.CluserConfig{ + Cluster_name: "test01.cern.ch", + Loadbalancing_username: "loadbalancing", + Loadbalancing_password: "zzz123", + }, logger) + host1.SetName("lxplus142.cern.ch") + host2 := lbhost.NewLBHost(model.CluserConfig{ + Cluster_name: "test01.cern.ch,test02.cern.ch", + Loadbalancing_username: "loadbalancing", + Loadbalancing_password: "zzz123", + }, logger) + host2.SetName("lxplus177.cern.ch") + host3 := lbhost.NewLBHost(model.CluserConfig{ + Cluster_name: "test02.cern.ch", + Loadbalancing_username: "loadbalancing", + Loadbalancing_password: "zzz123", + }, logger) + host3.SetName("lxplus013.cern.ch") + host4 := lbhost.NewLBHost(model.CluserConfig{ + Cluster_name: "test02.cern.ch", + Loadbalancing_username: "loadbalancing", + Loadbalancing_password: "zzz123", + }, logger) + host4.SetName("lxplus025.cern.ch") + expected := map[string]lbhost.Host{ + "lxplus142.cern.ch": host1, + "lxplus177.cern.ch": host2, + "lxplus013.cern.ch": host3, + "lxplus025.cern.ch": host4, } - hosts_to_check := make(map[string]lbhost.LBHost) + hosts_to_check := make(map[string]lbhost.Host) for _, c := range clusters { c.Get_list_hosts(hosts_to_check) } diff --git a/tests/get_state_dns_test.go b/tests/get_state_dns_test.go index fe42b38..4a561bd 100644 --- a/tests/get_state_dns_test.go +++ b/tests/get_state_dns_test.go @@ -1,10 +1,9 @@ package main_test import ( + "lb-experts/golbd/lbcluster" "reflect" "testing" - - "gitlab.cern.ch/lb-experts/golbd/lbcluster" ) //TestGetStateDNS tests the function get_state_dns @@ -37,13 +36,13 @@ func TestGetStateDNS(t *testing.T) { iprecString = append(iprecString, ip.String()) } //Casting to string. The DeepEqual of IP is a bit tricky, since it can - received[c.Cluster_name] = []interface{}{iprecString, err} + received[c.ClusterConfig.Cluster_name] = []interface{}{iprecString, err} } //DeepEqual comparison between the map with expected values and the one with the outputs for _, c := range Clusters { - if !reflect.DeepEqual(received[c.Cluster_name], expected[c.Cluster_name]) { - t.Errorf("\ngot ips\n%T type and value %v\nexpected\n%T type and value %v", received[c.Cluster_name][0], received[c.Cluster_name][0], expected[c.Cluster_name][0], expected[c.Cluster_name][0]) - t.Errorf("\ngot error\n%T type and value %v\nexpected\n%T type and value %v", received[c.Cluster_name][1], received[c.Cluster_name][1], expected[c.Cluster_name][1], expected[c.Cluster_name][1]) + if !reflect.DeepEqual(received[c.ClusterConfig.Cluster_name], expected[c.ClusterConfig.Cluster_name]) { + t.Errorf("\ngot ips\n%T type and value %v\nexpected\n%T type and value %v", received[c.ClusterConfig.Cluster_name][0], received[c.ClusterConfig.Cluster_name][0], expected[c.ClusterConfig.Cluster_name][0], expected[c.ClusterConfig.Cluster_name][0]) + t.Errorf("\ngot error\n%T type and value %v\nexpected\n%T type and value %v", received[c.ClusterConfig.Cluster_name][1], received[c.ClusterConfig.Cluster_name][1], expected[c.ClusterConfig.Cluster_name][1], expected[c.ClusterConfig.Cluster_name][1]) } } diff --git a/tests/lbcluster_log_test.go b/tests/lbcluster_log_test.go index 98003cd..330143a 100644 --- a/tests/lbcluster_log_test.go +++ b/tests/lbcluster_log_test.go @@ -3,7 +3,7 @@ package main_test import ( "fmt" "io/ioutil" - "lb-experts/golbd/lbcluster" + "lb-experts/golbd/logger" "log" "os" "strings" @@ -20,7 +20,7 @@ func getLogFilePath() string { return fmt.Sprintf("%s/%s.%s", testLogDirPath, testFileName, "log") } func TestLBClusterLoggerForInitFailure(t *testing.T) { - _, err := lbcluster.NewLoggerFactory("") + _, err := logger.NewLoggerFactory("") if err == nil { t.Errorf("expected error not thrown") } @@ -28,7 +28,7 @@ func TestLBClusterLoggerForInitFailure(t *testing.T) { func TestLBClusterLoggerForInitSuccess(t *testing.T) { defer deleteFile(t) - logger, err := lbcluster.NewLoggerFactory(getLogFilePath()) + logger, err := logger.NewLoggerFactory(getLogFilePath()) if err != nil { t.Fail() t.Errorf("unexpected error thrown. error: %v", err) @@ -41,7 +41,7 @@ func TestLBClusterLoggerForInitSuccess(t *testing.T) { func TestLBClusterLoggerForSnapshot(t *testing.T) { defer deleteFile(t) - logger, err := lbcluster.NewLoggerFactory(getLogFilePath()) + logger, err := logger.NewLoggerFactory(getLogFilePath()) if err != nil { t.Fail() t.Errorf("unexpected error thrown. error: %v", err) @@ -60,7 +60,7 @@ func TestLBClusterLoggerForSnapshot(t *testing.T) { func TestLBClusterLoggerForNewSnapshot(t *testing.T) { defer deleteFile(t) - logger, err := lbcluster.NewLoggerFactory(getLogFilePath()) + logger, err := logger.NewLoggerFactory(getLogFilePath()) if err != nil { t.Fail() t.Errorf("unexpected error thrown. error: %v", err) @@ -83,7 +83,7 @@ func TestLBClusterLoggerForNewSnapshot(t *testing.T) { func TestLBClusterLoggerForDebugDisabled(t *testing.T) { defer deleteFile(t) - logger, err := lbcluster.NewLoggerFactory(getLogFilePath()) + logger, err := logger.NewLoggerFactory(getLogFilePath()) if err != nil { t.Fail() t.Errorf("unexpected error thrown. error: %v", err) @@ -97,7 +97,7 @@ func TestLBClusterLoggerForDebugDisabled(t *testing.T) { func TestLBClusterLoggerForDebugEnabled(t *testing.T) { defer deleteFile(t) - logger, err := lbcluster.NewLoggerFactory(getLogFilePath()) + logger, err := logger.NewLoggerFactory(getLogFilePath()) if err != nil { t.Fail() t.Errorf("unexpected error thrown. error: %v", err) @@ -112,7 +112,7 @@ func TestLBClusterLoggerForDebugEnabled(t *testing.T) { func TestLBClusterLoggerForInfo(t *testing.T) { defer deleteFile(t) - logger, err := lbcluster.NewLoggerFactory(getLogFilePath()) + logger, err := logger.NewLoggerFactory(getLogFilePath()) if err != nil { t.Fail() t.Errorf("unexpected error thrown. error: %v", err) @@ -126,7 +126,7 @@ func TestLBClusterLoggerForInfo(t *testing.T) { func TestLBClusterLoggerForWarning(t *testing.T) { defer deleteFile(t) - logger, err := lbcluster.NewLoggerFactory(getLogFilePath()) + logger, err := logger.NewLoggerFactory(getLogFilePath()) if err != nil { t.Fail() t.Errorf("unexpected error thrown. error: %v", err) @@ -140,7 +140,7 @@ func TestLBClusterLoggerForWarning(t *testing.T) { func TestLBClusterLoggerForError(t *testing.T) { defer deleteFile(t) - logger, err := lbcluster.NewLoggerFactory(getLogFilePath()) + logger, err := logger.NewLoggerFactory(getLogFilePath()) if err != nil { t.Fail() t.Errorf("unexpected error thrown. error: %v", err) diff --git a/tests/loadClusters_test.go b/tests/loadClusters_test.go index 2e4a034..4d8ef6e 100644 --- a/tests/loadClusters_test.go +++ b/tests/loadClusters_test.go @@ -1,20 +1,25 @@ package main_test import ( + "lb-experts/golbd/lbcluster" + "lb-experts/golbd/lbconfig" + "lb-experts/golbd/lbhost" + "lb-experts/golbd/logger" + "lb-experts/golbd/model" "net" "reflect" "testing" - - "gitlab.cern.ch/lb-experts/golbd/lbcluster" - "gitlab.cern.ch/lb-experts/golbd/lbconfig" - "gitlab.cern.ch/lb-experts/golbd/lbhost" ) func getTestCluster(name string) lbcluster.LBCluster { - lg := lbcluster.Log{SyslogWriter: nil, Stdout: true, Debugflag: false} - return lbcluster.LBCluster{Cluster_name: name, - Loadbalancing_username: "loadbalancing", - Loadbalancing_password: "zzz123", + lg, _ := logger.NewLoggerFactory("") + + return lbcluster.LBCluster{ + ClusterConfig: model.CluserConfig{ + Cluster_name: name, + Loadbalancing_username: "loadbalancing", + Loadbalancing_password: "zzz123", + }, Host_metric_table: map[string]lbcluster.Node{ "lxplus132.cern.ch": lbcluster.Node{Load: 100000, IPs: []net.IP{}}, "lxplus041.cern.ch": lbcluster.Node{Load: 100000, IPs: []net.IP{}}, @@ -25,15 +30,19 @@ func getTestCluster(name string) lbcluster.LBCluster { //Time_of_last_evaluation time.Time Current_best_ips: []net.IP{}, Previous_best_ips_dns: []net.IP{}, - Slog: &lg, + Slog: lg, Current_index: 0} } func getSecondTestCluster() lbcluster.LBCluster { - lg := lbcluster.Log{SyslogWriter: nil, Stdout: true, Debugflag: false} - return lbcluster.LBCluster{Cluster_name: "test02.test.cern.ch", - Loadbalancing_username: "loadbalancing", - Loadbalancing_password: "zzz123", + lg, _ := logger.NewLoggerFactory("") + + return lbcluster.LBCluster{ + ClusterConfig: model.CluserConfig{ + Cluster_name: "test02.test.cern.ch", + Loadbalancing_username: "loadbalancing", + Loadbalancing_password: "zzz123", + }, Host_metric_table: map[string]lbcluster.Node{ "lxplus013.cern.ch": lbcluster.Node{Load: 100000, IPs: []net.IP{}}, "lxplus038.cern.ch": lbcluster.Node{Load: 100000, IPs: []net.IP{}}, @@ -43,152 +52,130 @@ func getSecondTestCluster() lbcluster.LBCluster { //Time_of_last_evaluation time.Time Current_best_ips: []net.IP{}, Previous_best_ips_dns: []net.IP{}, - Slog: &lg, + Slog: lg, Current_index: 0} } -func getHostsToCheck(c lbcluster.LBCluster) map[string]lbhost.LBHost { - hostsToCheck := map[string]lbhost.LBHost{ - "lxplus132.cern.ch": lbhost.LBHost{Cluster_name: c.Cluster_name, - Host_name: "lxplus132.cern.ch", - Host_transports: []lbhost.LBHostTransportResult{ - lbhost.LBHostTransportResult{Transport: "udp6", Response_int: 2, Response_string: "", IP: net.ParseIP("2001:1458:d00:2c::100:a6"), Response_error: ""}, - lbhost.LBHostTransportResult{Transport: "udp", Response_int: 2, Response_string: "", IP: net.ParseIP("188.184.108.98"), Response_error: ""}, - }, - Loadbalancing_username: c.Loadbalancing_username, - Loadbalancing_password: c.Loadbalancing_password, - LogFile: c.Slog.TofilePath, - Debugflag: c.Slog.Debugflag, - }, - "lxplus041.cern.ch": lbhost.LBHost{Cluster_name: c.Cluster_name, - Host_name: "lxplus041.cern.ch", - Host_transports: []lbhost.LBHostTransportResult{ - lbhost.LBHostTransportResult{Transport: "udp6", Response_int: 3, Response_string: "", IP: net.ParseIP("2001:1458:d00:32::100:51"), Response_error: ""}, - lbhost.LBHostTransportResult{Transport: "udp", Response_int: 3, Response_string: "", IP: net.ParseIP("188.184.116.81"), Response_error: ""}, - }, - Loadbalancing_username: c.Loadbalancing_username, - Loadbalancing_password: c.Loadbalancing_password, - LogFile: c.Slog.TofilePath, - Debugflag: c.Slog.Debugflag, - }, - "lxplus130.cern.ch": lbhost.LBHost{Cluster_name: c.Cluster_name, - Host_name: "lxplus130.cern.ch", - Host_transports: []lbhost.LBHostTransportResult{lbhost.LBHostTransportResult{Transport: "udp", Response_int: 27, Response_string: "", IP: net.ParseIP("188.184.108.100"), Response_error: ""}}, - Loadbalancing_username: c.Loadbalancing_username, - Loadbalancing_password: c.Loadbalancing_password, - LogFile: c.Slog.TofilePath, - Debugflag: c.Slog.Debugflag, - }, - "lxplus133.subdo.cern.ch": lbhost.LBHost{Cluster_name: c.Cluster_name, - Host_name: "lxplus130.subdo.cern.ch", - Host_transports: []lbhost.LBHostTransportResult{lbhost.LBHostTransportResult{Transport: "udp", Response_int: 27, Response_string: "", IP: net.ParseIP("188.184.108.101"), Response_error: ""}}, - Loadbalancing_username: c.Loadbalancing_username, - Loadbalancing_password: c.Loadbalancing_password, - LogFile: c.Slog.TofilePath, - Debugflag: c.Slog.Debugflag, - }, - "monit-kafkax-17be060b0d.cern.ch": lbhost.LBHost{Cluster_name: c.Cluster_name, - Host_name: "monit-kafkax-17be060b0d.cern.ch", - Host_transports: []lbhost.LBHostTransportResult{lbhost.LBHostTransportResult{Transport: "udp", Response_int: 100000, Response_string: "monit-kafkax.cern.ch=816,monit-kafka.cern.ch=816,test01.cern.ch=816", IP: net.ParseIP("188.184.108.100"), Response_error: ""}}, - Loadbalancing_username: c.Loadbalancing_username, - Loadbalancing_password: c.Loadbalancing_password, - LogFile: c.Slog.TofilePath, - Debugflag: c.Slog.Debugflag, - }, +func getHostsToCheck(c lbcluster.LBCluster) map[string]lbhost.Host { + lg, _ := logger.NewLoggerFactory("") + host1 := lbhost.NewLBHost(c.ClusterConfig, lg) + host1.SetName("lxplus132.cern.ch") + host1.SetTransportPayload([]lbhost.LBHostTransportResult{ + lbhost.LBHostTransportResult{Transport: "udp6", Response_int: 2, Response_string: "", IP: net.ParseIP("2001:1458:d00:2c::100:a6"), Response_error: ""}, + lbhost.LBHostTransportResult{Transport: "udp", Response_int: 2, Response_string: "", IP: net.ParseIP("188.184.108.98"), Response_error: ""}, + }) + host2 := lbhost.NewLBHost(c.ClusterConfig, lg) + host2.SetName("lxplus041.cern.ch") + host2.SetTransportPayload([]lbhost.LBHostTransportResult{ + lbhost.LBHostTransportResult{Transport: "udp6", Response_int: 3, Response_string: "", IP: net.ParseIP("2001:1458:d00:32::100:51"), Response_error: ""}, + lbhost.LBHostTransportResult{Transport: "udp", Response_int: 3, Response_string: "", IP: net.ParseIP("188.184.116.81"), Response_error: ""}, + }) + host3 := lbhost.NewLBHost(c.ClusterConfig, lg) + host3.SetName("lxplus130.cern.ch") + host3.SetTransportPayload([]lbhost.LBHostTransportResult{lbhost.LBHostTransportResult{ + Transport: "udp", Response_int: 27, Response_string: "", IP: net.ParseIP("188.184.108.100"), Response_error: "", + }}) + host4 := lbhost.NewLBHost(c.ClusterConfig, lg) + host4.SetName("lxplus133.subdo.cern.ch") + host4.SetTransportPayload([]lbhost.LBHostTransportResult{lbhost.LBHostTransportResult{ + Transport: "udp", Response_int: 27, Response_string: "", IP: net.ParseIP("188.184.108.101"), Response_error: "", + }}) + host5 := lbhost.NewLBHost(c.ClusterConfig, lg) + host5.SetName("monit-kafkax-17be060b0d.cern.ch") + host5.SetTransportPayload([]lbhost.LBHostTransportResult{lbhost.LBHostTransportResult{ + Transport: "udp", Response_int: 100000, Response_string: "monit-kafkax.cern.ch=816,monit-kafka.cern.ch=816,test01.cern.ch=816", IP: net.ParseIP("188.184.108.100"), Response_error: ""}}, + ) + hostsToCheck := map[string]lbhost.Host{ + "lxplus132.cern.ch": host1, + "lxplus041.cern.ch": host2, + "lxplus130.cern.ch": host3, + "lxplus133.subdo.cern.ch": host4, + "monit-kafkax-17be060b0d.cern.ch": host5, } return hostsToCheck } -func getBadHostsToCheck(c lbcluster.LBCluster) map[string]lbhost.LBHost { - badHostsToCheck := map[string]lbhost.LBHost{ - "lxplus132.cern.ch": lbhost.LBHost{Cluster_name: c.Cluster_name, - Host_name: "lxplus132.cern.ch", - Host_transports: []lbhost.LBHostTransportResult{ - lbhost.LBHostTransportResult{Transport: "udp6", Response_int: -2, Response_string: "", IP: net.ParseIP("2001:1458:d00:2c::100:a6"), Response_error: ""}, - lbhost.LBHostTransportResult{Transport: "udp", Response_int: -2, Response_string: "", IP: net.ParseIP("188.184.108.98"), Response_error: ""}, - }, - Loadbalancing_username: c.Loadbalancing_username, - Loadbalancing_password: c.Loadbalancing_password, - LogFile: c.Slog.TofilePath, - Debugflag: c.Slog.Debugflag, - }, - "lxplus041.cern.ch": lbhost.LBHost{Cluster_name: c.Cluster_name, - Host_name: "lxplus041.cern.ch", - Host_transports: []lbhost.LBHostTransportResult{ - lbhost.LBHostTransportResult{Transport: "udp6", Response_int: -3, Response_string: "", IP: net.ParseIP("2001:1458:d00:32::100:51"), Response_error: ""}, - lbhost.LBHostTransportResult{Transport: "udp", Response_int: -3, Response_string: "", IP: net.ParseIP("188.184.116.81"), Response_error: ""}, - }, - Loadbalancing_username: c.Loadbalancing_username, - Loadbalancing_password: c.Loadbalancing_password, - LogFile: c.Slog.TofilePath, - Debugflag: c.Slog.Debugflag, - }, - "lxplus130.cern.ch": lbhost.LBHost{Cluster_name: c.Cluster_name, - Host_name: "lxplus130.cern.ch", - Host_transports: []lbhost.LBHostTransportResult{lbhost.LBHostTransportResult{Transport: "udp", Response_int: -27, Response_string: "", IP: net.ParseIP("188.184.108.100"), Response_error: ""}}, - Loadbalancing_username: c.Loadbalancing_username, - Loadbalancing_password: c.Loadbalancing_password, - LogFile: c.Slog.TofilePath, - Debugflag: c.Slog.Debugflag, - }, - "lxplus133.subdo.cern.ch": lbhost.LBHost{Cluster_name: c.Cluster_name, - Host_name: "lxplus133.subdo.cern.ch", - Host_transports: []lbhost.LBHostTransportResult{lbhost.LBHostTransportResult{Transport: "udp", Response_int: -15, Response_string: "", IP: net.ParseIP("188.184.108.101"), Response_error: ""}}, - Loadbalancing_username: c.Loadbalancing_username, - Loadbalancing_password: c.Loadbalancing_password, - LogFile: c.Slog.TofilePath, - Debugflag: c.Slog.Debugflag, - }, - "monit-kafkax-17be060b0d.cern.ch": lbhost.LBHost{Cluster_name: c.Cluster_name, - Host_name: "monit-kafkax-17be060b0d.cern.ch", - Host_transports: []lbhost.LBHostTransportResult{lbhost.LBHostTransportResult{Transport: "udp", Response_int: 100000, Response_string: "monit-kafkax.cern.ch=816,monit-kafka.cern.ch=816,test01.cern.ch=816", IP: net.ParseIP("188.184.108.100"), Response_error: ""}}, - Loadbalancing_username: c.Loadbalancing_username, - Loadbalancing_password: c.Loadbalancing_password, - LogFile: c.Slog.TofilePath, - Debugflag: c.Slog.Debugflag, - }, +func getBadHostsToCheck(c lbcluster.LBCluster) map[string]lbhost.Host { + lg, _ := logger.NewLoggerFactory("") + host1 := lbhost.NewLBHost(c.ClusterConfig, lg) + host1.SetName("lxplus132.cern.ch") + host1.SetTransportPayload([]lbhost.LBHostTransportResult{ + lbhost.LBHostTransportResult{Transport: "udp6", Response_int: 2, Response_string: "", IP: net.ParseIP("2001:1458:d00:2c::100:a6"), Response_error: ""}, + lbhost.LBHostTransportResult{Transport: "udp", Response_int: 2, Response_string: "", IP: net.ParseIP("188.184.108.98"), Response_error: ""}, + }) + host2 := lbhost.NewLBHost(c.ClusterConfig, lg) + host2.SetName("lxplus041.cern.ch") + host2.SetTransportPayload([]lbhost.LBHostTransportResult{ + lbhost.LBHostTransportResult{Transport: "udp6", Response_int: 3, Response_string: "", IP: net.ParseIP("2001:1458:d00:32::100:51"), Response_error: ""}, + lbhost.LBHostTransportResult{Transport: "udp", Response_int: 3, Response_string: "", IP: net.ParseIP("188.184.116.81"), Response_error: ""}, + }) + host3 := lbhost.NewLBHost(c.ClusterConfig, lg) + host3.SetName("lxplus130.cern.ch") + host3.SetTransportPayload([]lbhost.LBHostTransportResult{lbhost.LBHostTransportResult{ + Transport: "udp", Response_int: 27, Response_string: "", IP: net.ParseIP("188.184.108.100"), Response_error: "", + }}) + host4 := lbhost.NewLBHost(c.ClusterConfig, lg) + host4.SetName("lxplus133.subdo.cern.ch") + host4.SetTransportPayload([]lbhost.LBHostTransportResult{lbhost.LBHostTransportResult{ + Transport: "udp", Response_int: 27, Response_string: "", IP: net.ParseIP("188.184.108.101"), Response_error: "", + }}) + host5 := lbhost.NewLBHost(c.ClusterConfig, lg) + host5.SetName("monit-kafkax-17be060b0d.cern.ch") + host5.SetTransportPayload([]lbhost.LBHostTransportResult{lbhost.LBHostTransportResult{ + Transport: "udp", Response_int: 100000, Response_string: "monit-kafkax.cern.ch=816,monit-kafka.cern.ch=816,test01.cern.ch=816", IP: net.ParseIP("188.184.108.100"), Response_error: ""}}, + ) + badHostsToCheck := map[string]lbhost.Host{ + "lxplus132.cern.ch": host1, + "lxplus041.cern.ch": host2, + "lxplus130.cern.ch": host3, + "lxplus133.subdo.cern.ch": host4, + "monit-kafkax-17be060b0d.cern.ch": host5, } return badHostsToCheck } -func getHost(hostname string, responseInt int, responseString string) lbhost.LBHost { - - return lbhost.LBHost{Cluster_name: "test01.cern.ch", - Host_name: hostname, - Host_transports: []lbhost.LBHostTransportResult{ - lbhost.LBHostTransportResult{Transport: "udp", Response_int: responseInt, Response_string: responseString, IP: net.ParseIP("188.184.108.98"), Response_error: ""}}, +func getHost(hostname string, responseInt int, responseString string) lbhost.Host { + lg, _ := logger.NewLoggerFactory("") + clusterConfig := model.CluserConfig{ + Cluster_name: "test01.cern.ch", Loadbalancing_username: "loadbalancing", Loadbalancing_password: "XXXX", - LogFile: "", - Debugflag: false, } + host1 := lbhost.NewLBHost(clusterConfig, lg) + host1.SetName(hostname) + host1.SetTransportPayload([]lbhost.LBHostTransportResult{ + lbhost.LBHostTransportResult{Transport: "udp", Response_int: responseInt, Response_string: responseString, IP: net.ParseIP("188.184.108.98"), Response_error: ""}}, + ) + return host1 } func TestLoadClusters(t *testing.T) { - lg := lbcluster.Log{SyslogWriter: nil, Stdout: true, Debugflag: false} + lg, _ := logger.NewLoggerFactory("") + lg.EnableWriteToSTd() + + config := lbconfig.NewLoadBalancerConfig("", lg) + config.SetMasterHost("lbdxyz.cern.ch") + config.SetHeartBeatFileName("heartbeat") + config.SetHeartBeatDirPath("/work/go/src/github.com/cernops/golbd") + config.SetTSIGKeyPrefix("abcd-") + config.SetTSIGInternalKey("xxx123==") + config.SetTSIGExternalKey("yyy123==") + config.SetDNSManager("111.111.0.111") + config.SetSNMPPassword("zzz123") + config.SetClusters(map[string][]string{"test01.cern.ch": {"lxplus132.cern.ch", "lxplus041.cern.ch", "lxplus130.cern.ch", "lxplus133.subdo.cern.ch", "monit-kafkax-17be060b0d.cern.ch"}, "test02.test.cern.ch": {"lxplus013.cern.ch", "lxplus038.cern.ch", "lxplus039.test.cern.ch", "lxplus025.cern.ch"}}) + config.SetParameters(map[string]lbcluster.Params{"test01.cern.ch": lbcluster.Params{Behaviour: "mindless", Best_hosts: 2, + External: true, Metric: "cmsfrontier", Polling_interval: 6, Statistics: "long"}, + "test02.test.cern.ch": lbcluster.Params{Behaviour: "mindless", Best_hosts: 10, External: false, Metric: "cmsfrontier", Polling_interval: 6, Statistics: "long"}}) - config := lbconfig.Config{Master: "lbdxyz.cern.ch", - HeartbeatFile: "heartbeat", - HeartbeatPath: "/work/go/src/github.com/cernops/golbd", - //HeartbeatMu: sync.Mutex{0, 0}, - TsigKeyPrefix: "abcd-", - TsigInternalKey: "xxx123==", - TsigExternalKey: "yyy123==", - SnmpPassword: "zzz123", - DNSManager: "111.111.0.111", - Clusters: map[string][]string{"test01.cern.ch": {"lxplus132.cern.ch", "lxplus041.cern.ch", "lxplus130.cern.ch", "lxplus133.subdo.cern.ch", "monit-kafkax-17be060b0d.cern.ch"}, "test02.test.cern.ch": {"lxplus013.cern.ch", "lxplus038.cern.ch", "lxplus039.test.cern.ch", "lxplus025.cern.ch"}}, - Parameters: map[string]lbcluster.Params{"test01.cern.ch": lbcluster.Params{Behaviour: "mindless", Best_hosts: 2, - External: true, Metric: "cmsfrontier", Polling_interval: 6, Statistics: "long"}, - "test02.test.cern.ch": lbcluster.Params{Behaviour: "mindless", Best_hosts: 10, External: false, Metric: "cmsfrontier", Polling_interval: 6, Statistics: "long"}}} expected := []lbcluster.LBCluster{getTestCluster("test01.cern.ch"), getSecondTestCluster()} - lbclusters, _ := lbconfig.LoadClusters(&config, &lg) + _, lbclusters, _ := config.Load() // reflect.DeepEqual(lbclusters, expected) occassionally fails as the array order is not always the same // so comparing element par element i := 0 for _, e := range expected { for _, c := range lbclusters { - if c.Cluster_name == e.Cluster_name { + if c.ClusterConfig.Cluster_name == e.ClusterConfig.Cluster_name { if !reflect.DeepEqual(c, e) { t.Errorf("loadClusters: got\n%v\nexpected\n%v", lbclusters, expected) } else { diff --git a/tests/loadConfig_test.go b/tests/loadConfig_test.go index d0104d6..e09c49a 100644 --- a/tests/loadConfig_test.go +++ b/tests/loadConfig_test.go @@ -1,10 +1,10 @@ package main import ( - lbconfig2 "lb-experts/golbd/lbconfig" + //lbconfig2 "lb-experts/golbd/lbconfig" "os" "reflect" - "sync" + //"sync" "testing" "gitlab.cern.ch/lb-experts/golbd/lbcluster" @@ -61,14 +61,14 @@ func TestLoadConfig(t *testing.T) { } -func TestWatchConfigFileChanges(t *testing.T) { - lg := lbcluster.Log{Stdout: true, Debugflag: false} - var wg *sync.WaitGroup - var controlChan = make(chan bool) - defer close(controlChan) - config:=lbconfig2.NewLoadBalancerConfig("testloadconfig", &lg) - fileChangeSignal := config.WatchFileChange(controlChan, wg) - for filChangeData := range fileChangeSignal { - - } -} +//func TestWatchConfigFileChanges(t *testing.T) { +// lg := lbcluster.Log{Stdout: true, Debugflag: false} +// var wg *sync.WaitGroup +// var controlChan = make(chan bool) +// defer close(controlChan) +// config:=lbconfig2.NewLoadBalancerConfig("testloadconfig", &lg) +// fileChangeSignal := config.WatchFileChange(controlChan, wg) +// for filChangeData := range fileChangeSignal { +// +// } +//} diff --git a/tests/sample.log b/tests/sample.log new file mode 100644 index 0000000..e69de29 From 073434ff19d68292ff4261caee1239cb4fa246f5 Mon Sep 17 00:00:00 2001 From: Yadu Nandan Date: Thu, 12 May 2022 18:33:49 -0700 Subject: [PATCH 11/20] test refractor and refractor get hosts --- lbcluster/lbcluster.go | 10 ++-- lbd.go | 5 +- lbhost/lbhost.go | 64 ++++++++++++++++------ tests/get_list_hosts_test.go | 19 ++++--- tests/loadClusters_test.go | 12 ++-- tests/loadConfig_test.go | 82 ++++++++++++---------------- tests/sample.log | 0 tests/sample_2022_May_12-18_31_1.log | 1 + 8 files changed, 109 insertions(+), 84 deletions(-) delete mode 100644 tests/sample.log create mode 100644 tests/sample_2022_May_12-18_31_1.log diff --git a/lbcluster/lbcluster.go b/lbcluster/lbcluster.go index a05d5ed..cd64c1f 100644 --- a/lbcluster/lbcluster.go +++ b/lbcluster/lbcluster.go @@ -94,19 +94,21 @@ func (lbc *LBCluster) Time_to_refresh() bool { return lbc.Time_of_last_evaluation.Add(time.Duration(lbc.Parameters.Polling_interval) * time.Second).Before(time.Now()) } -//Get_list_hosts Get the hosts for an alias -func (lbc *LBCluster) Get_list_hosts(current_list map[string]lbhost.Host) { +//GetHostList Get the hosts for an alias +func (lbc *LBCluster) GetHostList() map[string]lbhost.Host { + hostMap := make(map[string]lbhost.Host) lbc.Slog.Debug("Getting the list of hosts for the alias") for host := range lbc.Host_metric_table { - myHost, ok := current_list[host] + myHost, ok := hostMap[host] if ok { clusterConfig := myHost.GetClusterConfig() clusterConfig.Cluster_name = clusterConfig.Cluster_name + "," + clusterConfig.Cluster_name } else { myHost = lbhost.NewLBHost(lbc.ClusterConfig, lbc.Slog) } - current_list[host] = myHost + hostMap[host] = myHost } + return hostMap } func (lbc *LBCluster) concatenateNodes(myNodes []Node) string { diff --git a/lbd.go b/lbd.go index b31aeb8..0a71510 100644 --- a/lbd.go +++ b/lbd.go @@ -202,7 +202,6 @@ func main() { } } -// todo: add some tests func checkAliases(config lbconfig.Config, lg logger.Logger, lbclusters []lbcluster.LBCluster) { hostCheckChannel := make(chan lbhost.Host) defer close(hostCheckChannel) @@ -215,7 +214,7 @@ func checkAliases(config lbconfig.Config, lg logger.Logger, lbclusters []lbclust //var wg sync.WaitGroup updateDNS := true lg.Info("Checking if any of the " + strconv.Itoa(len(lbclusters)) + " clusters needs updating") - hostsToCheck := make(map[string]lbhost.Host) + var hostsToCheck map[string]lbhost.Host var clustersToUpdate []*lbcluster.LBCluster /* First, let's identify the hosts that have to be checked */ for i := range lbclusters { @@ -223,7 +222,7 @@ func checkAliases(config lbconfig.Config, lg logger.Logger, lbclusters []lbclust lg.Debug("DO WE HAVE TO UPDATE?") if currentCluster.Time_to_refresh() { lg.Info("Time to refresh the cluster") - currentCluster.Get_list_hosts(hostsToCheck) + hostsToCheck = currentCluster.GetHostList() clustersToUpdate = append(clustersToUpdate, currentCluster) } } diff --git a/lbhost/lbhost.go b/lbhost/lbhost.go index 97ade1b..8bbf302 100644 --- a/lbhost/lbhost.go +++ b/lbhost/lbhost.go @@ -7,6 +7,7 @@ import ( "net" "regexp" "strconv" + "sync" "time" "github.com/reguero/go-snmplib" @@ -32,6 +33,11 @@ type LBHost struct { Logger logger.Logger } +type snmpDiscoveryResult struct { + hostIdx int + hostTransportResult LBHostTransportResult +} + type Host interface { GetName() string SetName(name string) @@ -71,31 +77,53 @@ func (lh *LBHost) SetTransportPayload(transportPayloadList []LBHostTransportResu lh.HostTransports = transportPayloadList } -// todo: refractor into smaller functions func (lh *LBHost) SNMPDiscovery() { + var wg sync.WaitGroup lh.find_transports() + discoveryResultChan := make(chan snmpDiscoveryResult) + defer close(discoveryResultChan) + hostTransportResultList := make([]LBHostTransportResult, 0, len(lh.HostTransports)) + hostTransportResultList = append(hostTransportResultList, lh.HostTransports...) for i, hostTransport := range lh.HostTransports { - hostTransport.Response_int = DefaultResponseInt - node_ip := hostTransport.IP.String() - /* There is no need to put square brackets around the ipv6 addresses*/ - lh.Logger.Debug("Checking the host " + node_ip + " with " + hostTransport.Transport) - snmp, err := snmplib.NewSNMPv3(node_ip, lh.ClusterConfig.Loadbalancing_username, "MD5", lh.ClusterConfig.Loadbalancing_password, "NOPRIV", lh.ClusterConfig.Loadbalancing_password, - time.Duration(TIMEOUT)*time.Second, 2) + wg.Add(1) + go func(idx int, hostTransport LBHostTransportResult) { + defer wg.Done() + lh.discoverNode(idx, hostTransport, discoveryResultChan) + }(i, hostTransport) + } + go func(discoveryResultChan <-chan snmpDiscoveryResult) { + for discoveryResultData := range discoveryResultChan { + hostTransportResultList[discoveryResultData.hostIdx] = discoveryResultData.hostTransportResult + } + }(discoveryResultChan) + wg.Wait() + lh.HostTransports = hostTransportResultList + lh.Logger.Debug("All the ips have been tested") +} + +func (lh *LBHost) discoverNode(hostTransportIdx int, hostTransport LBHostTransportResult, resultChan chan<- snmpDiscoveryResult) { + hostTransport.Response_int = DefaultResponseInt + nodeIp := hostTransport.IP.String() + + lh.Logger.Debug("Checking the host " + nodeIp + " with " + hostTransport.Transport) + snmp, err := snmplib.NewSNMPv3(nodeIp, lh.ClusterConfig.Loadbalancing_username, "MD5", + lh.ClusterConfig.Loadbalancing_password, "NOPRIV", lh.ClusterConfig.Loadbalancing_password, + time.Duration(TIMEOUT)*time.Second, 2) + if err != nil { + hostTransport.Response_error = fmt.Sprintf("contacted node: error creating the snmp object: %v", err) + } else { + defer snmp.Close() + err = snmp.Discover() if err != nil { - hostTransport.Response_error = fmt.Sprintf("contacted node: error creating the snmp object: %v", err) + hostTransport.Response_error = fmt.Sprintf("contacted node: error in the snmp discovery: %v", err) } else { - defer snmp.Close() - err = snmp.Discover() - if err != nil { - hostTransport.Response_error = fmt.Sprintf("contacted node: error in the snmp discovery: %v", err) - } else { - lh.setTransportResponse(snmp, &hostTransport) - } + lh.setTransportResponse(snmp, &hostTransport) } - lh.HostTransports[i] = hostTransport - } - lh.Logger.Debug("All the ips have been tested") + resultChan <- snmpDiscoveryResult{ + hostIdx: hostTransportIdx, + hostTransportResult: hostTransport, + } } func (lh *LBHost) setTransportResponse(snmpClient *snmplib.SNMP, lbHostTransportResultPayload *LBHostTransportResult) { diff --git a/tests/get_list_hosts_test.go b/tests/get_list_hosts_test.go index 2d82a48..ed0dde1 100644 --- a/tests/get_list_hosts_test.go +++ b/tests/get_list_hosts_test.go @@ -6,6 +6,7 @@ import ( "lb-experts/golbd/logger" "lb-experts/golbd/model" "net" + "os" "reflect" "testing" ) @@ -30,15 +31,14 @@ func TestGetListHostsOne(t *testing.T) { "lxplus130.cern.ch": host5, } - hosts_to_check := make(map[string]lbhost.Host) - c.Get_list_hosts(hosts_to_check) - if !reflect.DeepEqual(hosts_to_check, expected) { - t.Errorf("e.Get_list_hosts: got\n%v\nexpected\n%v", hosts_to_check, expected) + hostsToCheck := c.GetHostList() + if !reflect.DeepEqual(hostsToCheck, expected) { + t.Errorf("e.GetHostList: got\n%v\nexpected\n%v", hostsToCheck, expected) } } func TestGetListHostsTwo(t *testing.T) { - logger, _ := logger.NewLoggerFactory("") + logger, _ := logger.NewLoggerFactory("sample.log") logger.EnableWriteToSTd() clusters := []lbcluster.LBCluster{ @@ -98,11 +98,12 @@ func TestGetListHostsTwo(t *testing.T) { "lxplus025.cern.ch": host4, } - hosts_to_check := make(map[string]lbhost.Host) + var hostsToCheck map[string]lbhost.Host for _, c := range clusters { - c.Get_list_hosts(hosts_to_check) + hostsToCheck = c.GetHostList() } - if !reflect.DeepEqual(hosts_to_check, expected) { - t.Errorf("e.Get_list_hosts: got\n%v\nexpected\n%v", hosts_to_check, expected) + if !reflect.DeepEqual(hostsToCheck, expected) { + t.Errorf("e.GetHostList: got\n%v\nexpected\n%v", hostsToCheck, expected) } + os.Remove("sample.log") } diff --git a/tests/loadClusters_test.go b/tests/loadClusters_test.go index 4d8ef6e..224f758 100644 --- a/tests/loadClusters_test.go +++ b/tests/loadClusters_test.go @@ -161,10 +161,14 @@ func TestLoadClusters(t *testing.T) { config.SetTSIGExternalKey("yyy123==") config.SetDNSManager("111.111.0.111") config.SetSNMPPassword("zzz123") - config.SetClusters(map[string][]string{"test01.cern.ch": {"lxplus132.cern.ch", "lxplus041.cern.ch", "lxplus130.cern.ch", "lxplus133.subdo.cern.ch", "monit-kafkax-17be060b0d.cern.ch"}, "test02.test.cern.ch": {"lxplus013.cern.ch", "lxplus038.cern.ch", "lxplus039.test.cern.ch", "lxplus025.cern.ch"}}) - config.SetParameters(map[string]lbcluster.Params{"test01.cern.ch": lbcluster.Params{Behaviour: "mindless", Best_hosts: 2, - External: true, Metric: "cmsfrontier", Polling_interval: 6, Statistics: "long"}, - "test02.test.cern.ch": lbcluster.Params{Behaviour: "mindless", Best_hosts: 10, External: false, Metric: "cmsfrontier", Polling_interval: 6, Statistics: "long"}}) + config.SetClusters(map[string][]string{ + "test01.cern.ch": {"lxplus132.cern.ch", "lxplus041.cern.ch", "lxplus130.cern.ch", "lxplus133.subdo.cern.ch", "monit-kafkax-17be060b0d.cern.ch"}, + "test02.test.cern.ch": {"lxplus013.cern.ch", "lxplus038.cern.ch", "lxplus039.test.cern.ch", "lxplus025.cern.ch"}, + }) + config.SetParameters(map[string]lbcluster.Params{ + "test01.cern.ch": lbcluster.Params{Behaviour: "mindless", Best_hosts: 2, External: true, Metric: "cmsfrontier", Polling_interval: 6, Statistics: "long"}, + "test02.test.cern.ch": lbcluster.Params{Behaviour: "mindless", Best_hosts: 10, External: false, Metric: "cmsfrontier", Polling_interval: 6, Statistics: "long"}, + }) expected := []lbcluster.LBCluster{getTestCluster("test01.cern.ch"), getSecondTestCluster()} diff --git a/tests/loadConfig_test.go b/tests/loadConfig_test.go index e09c49a..5773fec 100644 --- a/tests/loadConfig_test.go +++ b/tests/loadConfig_test.go @@ -1,64 +1,54 @@ package main import ( - //lbconfig2 "lb-experts/golbd/lbconfig" "os" "reflect" - //"sync" "testing" - "gitlab.cern.ch/lb-experts/golbd/lbcluster" - "gitlab.cern.ch/lb-experts/golbd/lbconfig" + "lb-experts/golbd/lbcluster" + "lb-experts/golbd/lbconfig" + "lb-experts/golbd/logger" ) func TestLoadConfig(t *testing.T) { - lg := lbcluster.Log{Stdout: true, Debugflag: false} - //open file - loadconfig, err := os.Open("testloadconfig") + lg, _ := logger.NewLoggerFactory("sample.log") + lg.EnableWriteToSTd() + + configFromFile := lbconfig.NewLoadBalancerConfig("testloadconfig", lg) + configExisting, _, err := configFromFile.Load() if err != nil { - panic(err) + t.Fail() + t.Errorf("loadConfig Error: %v", err.Error()) } - defer loadconfig.Close() - - // The expected output - expected := - lbconfig.Config{ - Master: "lbdxyz.cern.ch", - HeartbeatFile: "heartbeat", - HeartbeatPath: "/work/go/src/github.com/cernops/golbd", - //HeartbeatMu: sync.Mutex{0, 0}, - TsigKeyPrefix: "abcd-", - TsigInternalKey: "xxx123==", - TsigExternalKey: "yyy123==", - SnmpPassword: "zzz123", - DNSManager: "137.138.28.176", - ConfigFile: "testloadconfig", - Clusters: map[string][]string{ - "aiermis.cern.ch": {"ermis19.cern.ch", "ermis20.cern.ch"}, - "uermis.cern.ch": {"ermis21.cern.ch", "ermis22.cern.ch"}, - "permis.cern.ch": {"ermis21.sub.cern.ch", "ermis22.test.cern.ch", "ermis42.cern.ch"}, - "ermis.test.cern.ch": {"ermis23.cern.ch", "ermis24.cern.ch"}, - "ermis2.test.cern.ch": {"ermis23.toto.cern.ch", "ermis24.cern.ch", "ermis25.sub.cern.ch"}}, - Parameters: map[string]lbcluster.Params{ - "aiermis.cern.ch": {Behaviour: "mindless", Best_hosts: 1, External: false, Metric: "cmsfrontier", Polling_interval: 300, Statistics: "long", Ttl: 60}, - "uermis.cern.ch": {Behaviour: "mindless", Best_hosts: 1, External: false, Metric: "cmsfrontier", Polling_interval: 300, Statistics: "long", Ttl: 222}, - "permis.cern.ch": {Behaviour: "mindless", Best_hosts: 1, External: false, Metric: "cmsfrontier", Polling_interval: 300, Statistics: "long", Ttl: 222}, - "ermis.test.cern.ch": {Behaviour: "mindless", Best_hosts: 1, External: false, Metric: "cmsfrontier", Polling_interval: 300, Statistics: "long", Ttl: 222}, - "ermis2.test.cern.ch": {Behaviour: "mindless", Best_hosts: 1, External: false, Metric: "cmsfrontier", Polling_interval: 300, Statistics: "long", Ttl: 222}}} - - //retrieving the actual output - configExisting, _, e := lbconfig.LoadConfig(loadconfig.Name(), &lg) - - if e != nil { - t.Errorf("loadConfig Error: %v", e.Error()) - } else { - if !reflect.DeepEqual(configExisting, &expected) { - t.Errorf("loadConfig: got\n %v expected\n %v", configExisting, &expected) - } + expConfig := lbconfig.NewLoadBalancerConfig("", lg) + expConfig.SetMasterHost("lbdxyz.cern.ch") + expConfig.SetHeartBeatFileName("heartbeat") + expConfig.SetHeartBeatDirPath("/work/go/src/github.com/cernops/golbd") + expConfig.SetTSIGKeyPrefix("abcd-") + expConfig.SetTSIGInternalKey("xxx123==") + expConfig.SetTSIGExternalKey("yyy123==") + expConfig.SetDNSManager("137.138.28.176") + expConfig.SetSNMPPassword("zzz123") + expConfig.SetClusters(map[string][]string{ + "aiermis.cern.ch": {"ermis19.cern.ch", "ermis20.cern.ch"}, + "uermis.cern.ch": {"ermis21.cern.ch", "ermis22.cern.ch"}, + "permis.cern.ch": {"ermis21.sub.cern.ch", "ermis22.test.cern.ch", "ermis42.cern.ch"}, + "ermis.test.cern.ch": {"ermis23.cern.ch", "ermis24.cern.ch"}, + "ermis2.test.cern.ch": {"ermis23.toto.cern.ch", "ermis24.cern.ch", "ermis25.sub.cern.ch"}, + }) + expConfig.SetParameters(map[string]lbcluster.Params{ + "aiermis.cern.ch": {Behaviour: "mindless", Best_hosts: 1, External: false, Metric: "cmsfrontier", Polling_interval: 300, Statistics: "long", Ttl: 60}, + "uermis.cern.ch": {Behaviour: "mindless", Best_hosts: 1, External: false, Metric: "cmsfrontier", Polling_interval: 300, Statistics: "long", Ttl: 222}, + "permis.cern.ch": {Behaviour: "mindless", Best_hosts: 1, External: false, Metric: "cmsfrontier", Polling_interval: 300, Statistics: "long", Ttl: 222}, + "ermis.test.cern.ch": {Behaviour: "mindless", Best_hosts: 1, External: false, Metric: "cmsfrontier", Polling_interval: 300, Statistics: "long", Ttl: 222}, + "ermis2.test.cern.ch": {Behaviour: "mindless", Best_hosts: 1, External: false, Metric: "cmsfrontier", Polling_interval: 300, Statistics: "long", Ttl: 222}, + }) + if !reflect.DeepEqual(configExisting, &expConfig) { + t.Errorf("loadConfig: got\n %v expected\n %v", configExisting, &expConfig) } - + os.Remove("sample.log") } //func TestWatchConfigFileChanges(t *testing.T) { diff --git a/tests/sample.log b/tests/sample.log deleted file mode 100644 index e69de29..0000000 diff --git a/tests/sample_2022_May_12-18_31_1.log b/tests/sample_2022_May_12-18_31_1.log new file mode 100644 index 0000000..c38d11e --- /dev/null +++ b/tests/sample_2022_May_12-18_31_1.log @@ -0,0 +1 @@ +May 12 18:31:01.070 lbd[16106]: INFO: Clusters loaded From 166595e86255ce21fb6cb77ad74e0565d73c10ab Mon Sep 17 00:00:00 2001 From: Yadu Nandan Date: Thu, 12 May 2022 23:08:51 -0700 Subject: [PATCH 12/20] refractor and tests --- lbcluster/lbcluster.go | 10 +--- lbconfig/config.go | 47 ++++++++------- lbd.go | 14 ++--- lbhost/lbhost.go | 38 +++++++++---- logger/lbcluster_log.go | 5 +- tests/lbhost_test.go | 50 ++++++++++++++++ tests/loadClusters_test.go | 6 +- tests/loadConfig_test.go | 85 +++++++++++++++++++++++----- tests/sample_2022_May_12-18_31_1.log | 1 - 9 files changed, 188 insertions(+), 68 deletions(-) create mode 100644 tests/lbhost_test.go delete mode 100644 tests/sample_2022_May_12-18_31_1.log diff --git a/lbcluster/lbcluster.go b/lbcluster/lbcluster.go index cd64c1f..193f87b 100644 --- a/lbcluster/lbcluster.go +++ b/lbcluster/lbcluster.go @@ -4,8 +4,6 @@ import ( "encoding/json" "fmt" "io/ioutil" - "lb-experts/golbd/logger" - "lb-experts/golbd/model" "math/rand" "net" "net/http" @@ -15,17 +13,13 @@ import ( "time" "lb-experts/golbd/lbhost" + "lb-experts/golbd/logger" + "lb-experts/golbd/model" ) //WorstValue worst possible load const WorstValue int = 99999 -//TIMEOUT snmp timeout -const TIMEOUT int = 10 - -//OID snmp object to get -const OID string = ".1.3.6.1.4.1.96.255.1" - //LBCluster struct of an lbcluster alias type LBCluster struct { ClusterConfig model.CluserConfig diff --git a/lbconfig/config.go b/lbconfig/config.go index dbb8001..8b935f5 100644 --- a/lbconfig/config.go +++ b/lbconfig/config.go @@ -31,8 +31,8 @@ type Config interface { GetTSIGExternalKey() string LockHeartBeatMutex() UnlockHeartBeatMutex() - WatchFileChange(controlChan <-chan bool, waitGroup *sync.WaitGroup) <-chan ConfigFileChangeSignal - Load() (*LBConfig, []lbcluster.LBCluster, error) + WatchFileChange(controlChan <-chan bool, waitGroup sync.WaitGroup) <-chan ConfigFileChangeSignal + Load() ([]lbcluster.LBCluster, error) // testing only SetMasterHost(masterHostName string) @@ -157,7 +157,7 @@ func (c *LBConfig) UnlockHeartBeatMutex() { c.HeartbeatMu.Unlock() } -func (c *LBConfig) WatchFileChange(controlChan <-chan bool, waitGroup *sync.WaitGroup) <-chan ConfigFileChangeSignal { +func (c *LBConfig) WatchFileChange(controlChan <-chan bool, waitGroup sync.WaitGroup) <-chan ConfigFileChangeSignal { fileWatcherChan := make(chan ConfigFileChangeSignal) waitGroup.Add(1) go func() { @@ -189,17 +189,16 @@ func (c *LBConfig) WatchFileChange(controlChan <-chan bool, waitGroup *sync.Wait } //Load reads a configuration file and returns a struct with the config -func (c *LBConfig) Load() (*LBConfig, []lbcluster.LBCluster, error) { +func (c *LBConfig) Load() ([]lbcluster.LBCluster, error) { var ( - config LBConfig - p lbcluster.Params - mc = make(map[string][]string) - mp = make(map[string]lbcluster.Params) + p lbcluster.Params + mc = make(map[string][]string) + mp = make(map[string]lbcluster.Params) ) lines, err := readLines(c.configFilePath) if err != nil { - return nil, nil, err + return nil, err } for _, line := range lines { if strings.HasPrefix(line, "#") || (line == "") { @@ -209,21 +208,21 @@ func (c *LBConfig) Load() (*LBConfig, []lbcluster.LBCluster, error) { if words[1] == "=" { switch words[0] { case "master": - config.Master = words[2] + c.Master = words[2] case "heartbeat_path": - config.HeartbeatPath = words[2] + c.HeartbeatPath = words[2] case "heartbeat_file": - config.HeartbeatFile = words[2] + c.HeartbeatFile = words[2] case "tsig_key_prefix": - config.TsigKeyPrefix = words[2] + c.TsigKeyPrefix = words[2] case "tsig_internal_key": - config.TsigInternalKey = words[2] + c.TsigInternalKey = words[2] case "tsig_external_key": - config.TsigExternalKey = words[2] + c.TsigExternalKey = words[2] case "snmpd_password": - config.SnmpPassword = words[2] + c.SnmpPassword = words[2] case "dns_manager": - config.DNSManager = words[2] + c.DNSManager = words[2] } } else if words[2] == "=" { jsonStream := "{" @@ -261,17 +260,17 @@ func (c *LBConfig) Load() (*LBConfig, []lbcluster.LBCluster, error) { } } } - config.Parameters = mp - config.Clusters = mc + c.Parameters = mp + c.Clusters = mc - lbclusters, err := c.loadClusters() + lbclusters, err := c.LoadClusters() if err != nil { fmt.Println("Error getting the clusters") - return nil, nil, err + return nil, err } c.lbLog.Info("Clusters loaded") - return &config, lbclusters, nil + return lbclusters, nil } @@ -290,8 +289,8 @@ func readLines(path string) (lines []string, err error) { return lines, sc.Err() } -//loadClusters checks the syntax of the clusters defined in the configuration file -func (c *LBConfig) loadClusters() ([]lbcluster.LBCluster, error) { +//LoadClusters checks the syntax of the clusters defined in the configuration file +func (c *LBConfig) LoadClusters() ([]lbcluster.LBCluster, error) { var lbc lbcluster.LBCluster var lbcs []lbcluster.LBCluster diff --git a/lbd.go b/lbd.go index 0a71510..473481e 100644 --- a/lbd.go +++ b/lbd.go @@ -130,7 +130,7 @@ func updateHeartBeatToFile(heartBeatFilePath string, hostname string, lg logger. return nil } -func sleep(seconds time.Duration, controlChan <-chan bool, waitGroup *sync.WaitGroup) <-chan bool { +func sleep(seconds time.Duration, controlChan <-chan bool, waitGroup sync.WaitGroup) <-chan bool { sleepSignalChan := make(chan bool) waitGroup.Add(1) secondsTicker := time.NewTicker(seconds * time.Second) @@ -172,7 +172,7 @@ func main() { rand.Seed(time.Now().UTC().UnixNano()) logger.Info("Starting lbd") lbConfig := lbconfig.NewLoadBalancerConfig(*configFileFlag, logger) - config, lbclusters, err := lbConfig.Load() + lbclusters, err := lbConfig.Load() if err != nil { logger.Warning("loadConfig Error: ") logger.Warning(err.Error()) @@ -180,8 +180,8 @@ func main() { } logger.Info("Clusters loaded") - fileChangeSignal := lbConfig.WatchFileChange(controlChan, &wg) - intervalTickerSignal := sleep(DefaultSleepDuration, controlChan, &wg) + fileChangeSignal := lbConfig.WatchFileChange(controlChan, wg) + intervalTickerSignal := sleep(DefaultSleepDuration, controlChan, wg) for { select { case fileWatcherData := <-fileChangeSignal: @@ -190,13 +190,13 @@ func main() { controlChan <- true return } - logger.Info("CluserConfig Changed") - config, lbclusters, err = lbConfig.Load() + logger.Info("ClusterConfig Changed") + lbclusters, err = lbConfig.Load() if err != nil { logger.Error(fmt.Sprintf("Error getting the clusters (something wrong in %v", configFileFlag)) } case <-intervalTickerSignal: - checkAliases(config, logger, lbclusters) + checkAliases(lbConfig, logger, lbclusters) break } } diff --git a/lbhost/lbhost.go b/lbhost/lbhost.go index 8bbf302..54914e1 100644 --- a/lbhost/lbhost.go +++ b/lbhost/lbhost.go @@ -31,6 +31,7 @@ type LBHost struct { Host_name string HostTransports []LBHostTransportResult Logger logger.Logger + SnmpAgent DiscoveryAgent } type snmpDiscoveryResult struct { @@ -38,6 +39,18 @@ type snmpDiscoveryResult struct { hostTransportResult LBHostTransportResult } +type DiscoveryAgent interface { + Close() error + Discover() error + GetV3(oid snmplib.Oid) (interface{}, error) +} + +func NewHostDiscoveryAgent(nodeIp string, clusterConfig model.CluserConfig) (DiscoveryAgent, error) { + return snmplib.NewSNMPv3(nodeIp, clusterConfig.Loadbalancing_username, "MD5", + clusterConfig.Loadbalancing_password, "NOPRIV", clusterConfig.Loadbalancing_password, + time.Duration(TIMEOUT)*time.Second, 2) +} + type Host interface { GetName() string SetName(name string) @@ -102,31 +115,36 @@ func (lh *LBHost) SNMPDiscovery() { } func (lh *LBHost) discoverNode(hostTransportIdx int, hostTransport LBHostTransportResult, resultChan chan<- snmpDiscoveryResult) { + var snmpAgent DiscoveryAgent + var err error hostTransport.Response_int = DefaultResponseInt nodeIp := hostTransport.IP.String() - lh.Logger.Debug("Checking the host " + nodeIp + " with " + hostTransport.Transport) - snmp, err := snmplib.NewSNMPv3(nodeIp, lh.ClusterConfig.Loadbalancing_username, "MD5", - lh.ClusterConfig.Loadbalancing_password, "NOPRIV", lh.ClusterConfig.Loadbalancing_password, - time.Duration(TIMEOUT)*time.Second, 2) - if err != nil { - hostTransport.Response_error = fmt.Sprintf("contacted node: error creating the snmp object: %v", err) + if lh.SnmpAgent == nil { + snmpAgent, err = NewHostDiscoveryAgent(nodeIp, lh.ClusterConfig) + if err != nil { + hostTransport.Response_error = fmt.Sprintf("contacted node: error creating the snmp object: %v", err) + } } else { - defer snmp.Close() - err = snmp.Discover() + snmpAgent = lh.SnmpAgent + } + if err == nil { + defer snmpAgent.Close() + err = snmpAgent.Discover() if err != nil { hostTransport.Response_error = fmt.Sprintf("contacted node: error in the snmp discovery: %v", err) } else { - lh.setTransportResponse(snmp, &hostTransport) + lh.setTransportResponse(snmpAgent, &hostTransport) } } + resultChan <- snmpDiscoveryResult{ hostIdx: hostTransportIdx, hostTransportResult: hostTransport, } } -func (lh *LBHost) setTransportResponse(snmpClient *snmplib.SNMP, lbHostTransportResultPayload *LBHostTransportResult) { +func (lh *LBHost) setTransportResponse(snmpClient DiscoveryAgent, lbHostTransportResultPayload *LBHostTransportResult) { oid, err := snmplib.ParseOid(OID) if err != nil { lbHostTransportResultPayload.Response_error = fmt.Sprintf("contacted node: Error parsing the OID %v", err) diff --git a/logger/lbcluster_log.go b/logger/lbcluster_log.go index 584509d..4cfecab 100644 --- a/logger/lbcluster_log.go +++ b/logger/lbcluster_log.go @@ -116,7 +116,10 @@ func (l *Log) getFileNameTimeSuffix() string { } func (l *Log) shouldStartNewSnapshot() bool { - return time.Now().Sub(l.logStartTime) >= l.snapShotCycleTime + if l.isSnapShotEnabled { + return time.Now().Sub(l.logStartTime) >= l.snapShotCycleTime + } + return false } //Info write as Info diff --git a/tests/lbhost_test.go b/tests/lbhost_test.go new file mode 100644 index 0000000..572bb1a --- /dev/null +++ b/tests/lbhost_test.go @@ -0,0 +1,50 @@ +package main_test + +import ( + "github.com/reguero/go-snmplib" + "lb-experts/golbd/lbhost" + "lb-experts/golbd/logger" + "net" + "os" + "testing" + "time" +) + +type mockSNMPAgent struct { +} + +func (m mockSNMPAgent) Close() error { + return nil +} + +func (m mockSNMPAgent) Discover() error { + time.Sleep(1 * time.Second) + return nil +} + +func (m mockSNMPAgent) GetV3(oid snmplib.Oid) (interface{}, error) { + return 200, nil +} + +func NewMockSNMPAgent() lbhost.DiscoveryAgent { + return &mockSNMPAgent{} +} + +func TestSNMPDiscoveryForConcurrency(t *testing.T) { + lg, _ := logger.NewLoggerFactory("sample.log") + lg.EnableWriteToSTd() + host := lbhost.LBHost{Logger: lg, SnmpAgent: NewMockSNMPAgent()} + host.HostTransports = []lbhost.LBHostTransportResult{ + {IP: net.ParseIP("1.1.1.1"), Transport: "udp"}, + {IP: net.ParseIP("1.1.1.2"), Transport: "udp"}, + {IP: net.ParseIP("1.1.1.3"), Transport: "udp"}, + } + startTime := time.Now() + host.SNMPDiscovery() + endTime := time.Now() + if endTime.Sub(startTime) > 2*time.Second { + t.Fail() + t.Errorf("execution took more time than expected. expectedTime: %v, actualTime:%v", 1, endTime.Sub(startTime)) + } + os.Remove("sample.log") +} diff --git a/tests/loadClusters_test.go b/tests/loadClusters_test.go index 224f758..b534d83 100644 --- a/tests/loadClusters_test.go +++ b/tests/loadClusters_test.go @@ -7,6 +7,7 @@ import ( "lb-experts/golbd/logger" "lb-experts/golbd/model" "net" + "os" "reflect" "testing" ) @@ -149,7 +150,7 @@ func getHost(hostname string, responseInt int, responseString string) lbhost.Hos } func TestLoadClusters(t *testing.T) { - lg, _ := logger.NewLoggerFactory("") + lg, _ := logger.NewLoggerFactory("sample.log") lg.EnableWriteToSTd() config := lbconfig.NewLoadBalancerConfig("", lg) @@ -173,7 +174,7 @@ func TestLoadClusters(t *testing.T) { expected := []lbcluster.LBCluster{getTestCluster("test01.cern.ch"), getSecondTestCluster()} - _, lbclusters, _ := config.Load() + lbclusters, _ := config.Load() // reflect.DeepEqual(lbclusters, expected) occassionally fails as the array order is not always the same // so comparing element par element i := 0 @@ -193,4 +194,5 @@ func TestLoadClusters(t *testing.T) { t.Errorf("loadClusters: wrong number of clusters, got\n%v\nexpected\n%v (and %v", len(lbclusters), len(expected), i) } + os.Remove("sample.log") } diff --git a/tests/loadConfig_test.go b/tests/loadConfig_test.go index 5773fec..0efabf1 100644 --- a/tests/loadConfig_test.go +++ b/tests/loadConfig_test.go @@ -3,7 +3,9 @@ package main import ( "os" "reflect" + "sync" "testing" + "time" "lb-experts/golbd/lbcluster" "lb-experts/golbd/lbconfig" @@ -16,12 +18,12 @@ func TestLoadConfig(t *testing.T) { lg.EnableWriteToSTd() configFromFile := lbconfig.NewLoadBalancerConfig("testloadconfig", lg) - configExisting, _, err := configFromFile.Load() + _, err := configFromFile.Load() if err != nil { t.Fail() t.Errorf("loadConfig Error: %v", err.Error()) } - expConfig := lbconfig.NewLoadBalancerConfig("", lg) + expConfig := lbconfig.NewLoadBalancerConfig("testloadconfig", lg) expConfig.SetMasterHost("lbdxyz.cern.ch") expConfig.SetHeartBeatFileName("heartbeat") expConfig.SetHeartBeatDirPath("/work/go/src/github.com/cernops/golbd") @@ -45,20 +47,73 @@ func TestLoadConfig(t *testing.T) { "ermis2.test.cern.ch": {Behaviour: "mindless", Best_hosts: 1, External: false, Metric: "cmsfrontier", Polling_interval: 300, Statistics: "long", Ttl: 222}, }) - if !reflect.DeepEqual(configExisting, &expConfig) { - t.Errorf("loadConfig: got\n %v expected\n %v", configExisting, &expConfig) + if !reflect.DeepEqual(configFromFile, expConfig) { + t.Errorf("loadConfig: got\n %v expected\n %v", configFromFile, expConfig) } os.Remove("sample.log") } -//func TestWatchConfigFileChanges(t *testing.T) { -// lg := lbcluster.Log{Stdout: true, Debugflag: false} -// var wg *sync.WaitGroup -// var controlChan = make(chan bool) -// defer close(controlChan) -// config:=lbconfig2.NewLoadBalancerConfig("testloadconfig", &lg) -// fileChangeSignal := config.WatchFileChange(controlChan, wg) -// for filChangeData := range fileChangeSignal { -// -// } -//} +func TestWatchConfigFileChanges(t *testing.T) { + lg, _ := logger.NewLoggerFactory("sample.log") + lg.EnableWriteToSTd() + var wg sync.WaitGroup + var controlChan = make(chan bool) + var changeCounter int + sampleConfigFileName := "sampleConfig" + dataSet := []string{ + "data 1", + "data 12", + "data 123", + } + + err := createTestConfigFile(sampleConfigFileName) + if err != nil { + t.Fail() + t.Errorf("error while creating test config file. name: %s", sampleConfigFileName) + } + go func() { + defer close(controlChan) + for _, dataToWrite := range dataSet { + time.Sleep(1 * time.Second) + err = writeDataToFile(sampleConfigFileName, dataToWrite) + if err != nil { + t.Fail() + t.Errorf("error while writting to test config file. filename: %s, data:%s", sampleConfigFileName, dataToWrite) + } + } + }() + config := lbconfig.NewLoadBalancerConfig(sampleConfigFileName, lg) + fileChangeSignal := config.WatchFileChange(controlChan, wg) + for fileChangeData := range fileChangeSignal { + changeCounter += 1 + t.Log("file change signal", fileChangeData) + } + if changeCounter == 0 { + t.Fail() + t.Error("file changes not observed") + } + deleteFile("sample.log") + deleteFile(sampleConfigFileName) +} + +func createTestConfigFile(fileName string) error { + _, err := os.Create(fileName) + if err != nil { + return err + } + return nil +} + +func writeDataToFile(fileName string, data string) error { + fp, err := os.Create(fileName) + if err != nil { + return err + } + _, err = fp.WriteString(data) + fp.Close() + return err +} + +func deleteFile(fileName string) { + os.Remove(fileName) +} diff --git a/tests/sample_2022_May_12-18_31_1.log b/tests/sample_2022_May_12-18_31_1.log deleted file mode 100644 index c38d11e..0000000 --- a/tests/sample_2022_May_12-18_31_1.log +++ /dev/null @@ -1 +0,0 @@ -May 12 18:31:01.070 lbd[16106]: INFO: Clusters loaded From 8e9a2ffd68d7dec96a21e4dda3cbc9a82811494f Mon Sep 17 00:00:00 2001 From: Yadu Nandan Date: Sat, 14 May 2022 15:26:34 -0700 Subject: [PATCH 13/20] added hostmetric collection with api --- lbconfig/config.go | 35 ++++++----- metric/metric_controller.go | 55 ++++++++++++++++++ metric/metric_logic.go | 113 ++++++++++++++++++++++++++++++++++++ metric/metric_model.go | 67 +++++++++++++++++++++ tests/metric_test.go | 59 +++++++++++++++++++ 5 files changed, 315 insertions(+), 14 deletions(-) create mode 100644 metric/metric_controller.go create mode 100644 metric/metric_logic.go create mode 100644 metric/metric_model.go create mode 100644 tests/metric_test.go diff --git a/lbconfig/config.go b/lbconfig/config.go index 8b935f5..6881e1b 100644 --- a/lbconfig/config.go +++ b/lbconfig/config.go @@ -18,13 +18,15 @@ import ( ) const ( - DefaultLoadBalancerConfig = "loadbalancing" + DefaultLoadBalancerConfig = "loadbalancing" + DefaultMetricsDirectoryPath = "" ) type Config interface { GetMasterHost() string GetHeartBeatFileName() string GetHeartBeatDirPath() string + GetMetricDirectoryPath() string GetDNSManager() string GetTSIGKeyPrefix() string GetTSIGInternalKey() string @@ -49,19 +51,20 @@ type Config interface { // Config this is the configuration of the lbd type LBConfig struct { - Master string - HeartbeatFile string - HeartbeatPath string - HeartbeatMu sync.Mutex - TsigKeyPrefix string - TsigInternalKey string - TsigExternalKey string - SnmpPassword string - DNSManager string - configFilePath string - lbLog logger.Logger - Clusters map[string][]string - Parameters map[string]lbcluster.Params + Master string + HeartbeatFile string + HeartbeatPath string + HeartbeatMu sync.Mutex + TsigKeyPrefix string + TsigInternalKey string + TsigExternalKey string + SnmpPassword string + DNSManager string + configFilePath string + lbLog logger.Logger + Clusters map[string][]string + Parameters map[string]lbcluster.Params + metricDirectoryPath string } type ConfigFileChangeSignal struct { @@ -89,6 +92,10 @@ func (c *LBConfig) SetMasterHost(masterHostName string) { c.Master = masterHostName } +func (c *LBConfig) GetMetricDirectoryPath() string { + return c.metricDirectoryPath +} + func (c *LBConfig) GetHeartBeatFileName() string { return c.HeartbeatFile } diff --git a/metric/metric_controller.go b/metric/metric_controller.go new file mode 100644 index 0000000..827c777 --- /dev/null +++ b/metric/metric_controller.go @@ -0,0 +1,55 @@ +package metric + +import ( + "encoding/json" + "fmt" + "net/http" + "os" + "strings" +) + +const defaultPort = ":8081" + +type serverCtx struct { + metricLogic *BizLogic +} + +func StartNewMetricServer(metricDirectoryPath string) error { + currentHostname, err := os.Hostname() + if err != nil { + return fmt.Errorf("unable to fetch current host name. error: %v", err) + } + logic := NewLogic(metricDirectoryPath, currentHostname) + server := serverCtx{ + metricLogic: logic, + } + http.HandleFunc("/metric", server.fetchHostMetric) + err = http.ListenAndServe(defaultPort, nil) + if err != nil { + return err + } + return nil +} + +func (s *serverCtx) fetchHostMetric(resp http.ResponseWriter, req *http.Request) { + if !strings.EqualFold(req.Method, http.MethodGet) { + resp.WriteHeader(http.StatusBadRequest) + resp.Write([]byte("request method not supported")) + return + } + hostMetric, err := s.metricLogic.ReadHostMetric() + if err != nil { + resp.WriteHeader(http.StatusInternalServerError) + resp.Write([]byte(fmt.Sprintf("error while reading host metric. error: %v", err))) + return + } + responseData, err := json.Marshal(hostMetric) + if err != nil { + resp.WriteHeader(http.StatusInternalServerError) + resp.Write([]byte(fmt.Sprintf("error while marshalling data. error:%v", err))) + return + } + resp.Header().Set("content-type", "application/json") + resp.WriteHeader(http.StatusOK) + resp.Write(responseData) +} diff --git a/metric/metric_logic.go b/metric/metric_logic.go new file mode 100644 index 0000000..c459ca3 --- /dev/null +++ b/metric/metric_logic.go @@ -0,0 +1,113 @@ +package metric + +import ( + "encoding/csv" + "fmt" + "os" + "strings" + "sync" + "time" +) + +const ( + defaultMetricFileNamePrefix = "metric" + csvExtension = "csv" + defaultCycleDuration = 24 * time.Hour +) + +type BizLogic struct { + isCurrentHostMaster bool + hostName string + filePath string + dirPath string + rwLocker sync.RWMutex + recordStartTime time.Time +} + +func NewLogic(dirPath string, hostName string) *BizLogic { + logic := &BizLogic{ + dirPath: dirPath, + hostName: hostName, + } + logic.initNewFilePath() + return logic +} + +func (bl *BizLogic) ReadHostMetric() (HostMetric, error) { + propertyList, err := bl.readAllRecords() + if err != nil { + return HostMetric{}, err + } + return HostMetric{ + Name: bl.hostName, + PropertyList: propertyList, + }, nil +} + +func (bl *BizLogic) GetFilePath() string { + return bl.filePath +} + +func (bl *BizLogic) readAllRecords() ([]Property, error) { + var propertyResultList []Property + bl.rwLocker.RLock() + defer bl.rwLocker.RUnlock() + fp, err := os.Open(bl.filePath) + defer fp.Close() + if err != nil { + return propertyResultList, fmt.Errorf("error while reading metric. error: %v", err) + } + csvReader := csv.NewReader(fp) + + recordList, err := csvReader.ReadAll() + if err != nil { + return propertyResultList, fmt.Errorf("error while reading metric. error: %v", err) + } + for _, record := range recordList { + property, err := parseProperty(record) + if err != nil { + return propertyResultList, fmt.Errorf("error while parsing a property record. error: %s", err) + } + propertyResultList = append(propertyResultList, property) + } + return propertyResultList, nil +} + +func (bl *BizLogic) WriteRecord(property Property) error { + if err := property.validate(); err != nil { + return err + } + bl.rwLocker.Lock() + defer bl.rwLocker.Unlock() + if bl.shouldUpdateFilePath() { + bl.initNewFilePath() + } + property.RoundTripDuration = property.RoundTripEndTime.Sub(property.RoundTripStartTime) + if strings.EqualFold(bl.filePath, "") { + return fmt.Errorf("filePath cannot be empty") + } + + fp, err := os.OpenFile(bl.filePath, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0640) + defer fp.Close() + if err != nil { + return fmt.Errorf("error while recording metric. error: %v", err) + } + csvWriter := csv.NewWriter(fp) + err = csvWriter.Write(property.marshalToCSV()) + if err != nil { + return err + } + csvWriter.Flush() + + return nil +} + +func (bl *BizLogic) shouldUpdateFilePath() bool { + return time.Now().Sub(bl.recordStartTime) > defaultCycleDuration +} + +func (bl *BizLogic) initNewFilePath() { + curTime := time.Now() + bl.recordStartTime = curTime + bl.filePath = fmt.Sprintf("%s%s_%d_%d_%d.%s", bl.dirPath, defaultMetricFileNamePrefix, curTime.Year(), curTime.Month(), curTime.Day(), csvExtension) +} diff --git a/metric/metric_model.go b/metric/metric_model.go new file mode 100644 index 0000000..eea33bd --- /dev/null +++ b/metric/metric_model.go @@ -0,0 +1,67 @@ +package metric + +import ( + "fmt" + "time" +) + +type HostMetric struct { + Name string `json:"name" csv:"name" rw:"r"` + PropertyList []Property `json:"property_list" csv:"property_list" rw:"r"` +} + +type Property struct { + RoundTripStartTime time.Time `json:"start_time" csv:"start_time" rw:"r"` + RoundTripEndTime time.Time `json:"end_time" csv:"end_time" rw:"r"` + RoundTripDuration time.Duration `json:"round_trip_duration"` +} + +type ClusterMetric struct { + MasterName string + HostMetricList []HostMetric `json:"host_metric_list" csv:"host_metric_list" rw:"r"` +} + +func (p Property) validate() error { + var invalidFields []string + if p.RoundTripStartTime.IsZero() { + invalidFields = append(invalidFields, "start_time") + } + if p.RoundTripEndTime.IsZero() { + invalidFields = append(invalidFields, "end_time") + } + if len(invalidFields) != 0 { + return fmt.Errorf("following fields are not valid. %v", invalidFields) + } + return nil +} + +func (p Property) marshalToCSV() []string { + return []string{ + p.RoundTripStartTime.Format(time.RFC3339), + p.RoundTripEndTime.Format(time.RFC3339), + p.RoundTripDuration.String(), + } +} + +func parseProperty(csvRecord []string) (Property, error) { + if len(csvRecord) < 3 { + return Property{}, fmt.Errorf("insufficient columns. column count %d", len(csvRecord)) + } + parsedStartTime, err := time.Parse(time.RFC3339, csvRecord[0]) + if err != nil { + return Property{}, err + } + parsedEndTime, err := time.Parse(time.RFC3339, csvRecord[1]) + if err != nil { + return Property{}, err + } + parsedDuration, err := time.ParseDuration(csvRecord[2]) + if err != nil { + return Property{}, err + } + return Property{ + RoundTripStartTime: parsedStartTime, + RoundTripEndTime: parsedEndTime, + RoundTripDuration: parsedDuration, + }, nil +} diff --git a/tests/metric_test.go b/tests/metric_test.go new file mode 100644 index 0000000..4752770 --- /dev/null +++ b/tests/metric_test.go @@ -0,0 +1,59 @@ +package main_test + +import ( + "lb-experts/golbd/metric" + "os" + "strings" + "testing" + "time" +) + +func TestMetricReadWriteRecord(t *testing.T) { + hostName, _ := os.Hostname() + logic := metric.NewLogic("", hostName) + curTime := time.Now() + property := metric.Property{ + RoundTripStartTime: curTime, + RoundTripEndTime: curTime.Add(5 * time.Second), + RoundTripDuration: 5 * time.Second, + } + err := logic.WriteRecord(property) + if err != nil { + t.Fail() + t.Errorf("error while recoding metric. error:%v", err) + return + } + hostMetric, err := logic.ReadHostMetric() + if err != nil { + t.Fail() + t.Errorf("error while reading metric.error:%v", err) + return + } + + if hostMetric.PropertyList == nil || len(hostMetric.PropertyList) == 0 { + t.Fail() + t.Errorf("property list is empty. expected length :%d", 1) + return + } + expectedStarttime := property.RoundTripStartTime.Format(time.RFC3339) + expectedEndtime := property.RoundTripEndTime.Format(time.RFC3339) + actualStartTime := hostMetric.PropertyList[0].RoundTripStartTime.Format(time.RFC3339) + actualEndTime := hostMetric.PropertyList[0].RoundTripEndTime.Format(time.RFC3339) + if !strings.EqualFold(expectedStarttime, actualStartTime) { + t.Fail() + t.Errorf("start time value mismatch expected: %v, actual:%v", expectedStarttime, actualStartTime) + } + if !strings.EqualFold(expectedEndtime, actualEndTime) { + t.Fail() + t.Errorf("end time value mismatch expected: %v, actual:%v", expectedEndtime, actualEndTime) + } + if hostMetric.PropertyList[0].RoundTripDuration != property.RoundTripDuration { + t.Fail() + t.Errorf("duration value mismatch expected: %v, actual:%v", property.RoundTripDuration, hostMetric.PropertyList[0].RoundTripDuration) + } + err = os.Remove(logic.GetFilePath()) + if err != nil { + t.Fail() + t.Errorf("error deleting file.error %v", err) + } +} From ca72f011059c1495cdc8e9501b4df5f87812a704 Mon Sep 17 00:00:00 2001 From: Yadu Nandan Date: Sat, 14 May 2022 15:27:10 -0700 Subject: [PATCH 14/20] fixes for ut --- lbcluster/lbcluster.go | 2 +- tests/aliasload_test.go | 6 ++++++ tests/apply_metric_internal_test.go | 6 ++++++ tests/evaluate_hosts_internal_test.go | 5 +++++ tests/find_best_hosts_test.go | 8 +++++++- tests/get_list_hosts_test.go | 10 ++++++++-- tests/loadClusters_test.go | 28 +++++++++++++++------------ tests/sample.log | 22 +++++++++++++++++++++ 8 files changed, 71 insertions(+), 16 deletions(-) create mode 100644 tests/sample.log diff --git a/lbcluster/lbcluster.go b/lbcluster/lbcluster.go index 193f87b..5a61c16 100644 --- a/lbcluster/lbcluster.go +++ b/lbcluster/lbcluster.go @@ -307,7 +307,7 @@ func (lbc *LBCluster) EvaluateHosts(hostsToCheck map[string]lbhost.Host) { } go func() { for nodeData := range nodeChan { - lbc.Host_metric_table[nodeData.HostName] = nodeData + lbc.Host_metric_table[nodeData.HostName] = Node{Load: nodeData.Load, IPs: nodeData.IPs} lbc.Slog.Debug(fmt.Sprintf("node: %s It has a load of %d", nodeData.HostName, lbc.Host_metric_table[nodeData.HostName].Load)) wg.Done() } diff --git a/tests/aliasload_test.go b/tests/aliasload_test.go index 9bac85c..847cf75 100644 --- a/tests/aliasload_test.go +++ b/tests/aliasload_test.go @@ -2,6 +2,7 @@ package main_test import ( "lb-experts/golbd/lbhost" + "os" "reflect" "testing" ) @@ -45,4 +46,9 @@ func TestGetLoadHosts(t *testing.T) { if !reflect.DeepEqual(hosts[3].GetLoadForAlias("blablabla2.subdo.cern.ch"), 4) { t.Errorf(" got\n%v\nexpected\n%v", hosts[3].GetLoadForAlias("blablabla2.subdo.cern.ch"), 4) } + err := os.Remove("sample.log") + if err != nil { + t.Fail() + t.Errorf("error deleting file.error %v", err) + } } diff --git a/tests/apply_metric_internal_test.go b/tests/apply_metric_internal_test.go index 284e590..10064a6 100644 --- a/tests/apply_metric_internal_test.go +++ b/tests/apply_metric_internal_test.go @@ -3,6 +3,7 @@ package main_test import ( "fmt" "net" + "os" "reflect" "testing" ) @@ -49,4 +50,9 @@ func TestEvaluateMetric(t *testing.T) { if !reflect.DeepEqual(c.Time_of_last_evaluation, expected_time_of_last_evaluation) { t.Errorf("e.apply_metric: c.Time_of_last_evaluation: got\n%v\nexpected\n%v", c.Time_of_last_evaluation, expected_time_of_last_evaluation) } + err := os.Remove("sample.log") + if err != nil { + t.Fail() + t.Errorf("error deleting file.error %v", err) + } } diff --git a/tests/evaluate_hosts_internal_test.go b/tests/evaluate_hosts_internal_test.go index 9b3a9fb..40b8f7f 100644 --- a/tests/evaluate_hosts_internal_test.go +++ b/tests/evaluate_hosts_internal_test.go @@ -120,6 +120,11 @@ func TestEvaluateHosts(t *testing.T) { if !reflect.DeepEqual(c.Time_of_last_evaluation, expectedTimeOfLastEvaluation) { t.Errorf("e.evaluate_hosts: c.Time_of_last_evaluation: got\n%v\nexpected\n%v", c.Time_of_last_evaluation, expectedTimeOfLastEvaluation) } + err := os.Remove("sample.log") + if err != nil { + t.Fail() + t.Errorf("error deleting file.error %v", err) + } } func TestEvaluateHostsConcurrency(t *testing.T) { diff --git a/tests/find_best_hosts_test.go b/tests/find_best_hosts_test.go index ea2cfd2..745e348 100644 --- a/tests/find_best_hosts_test.go +++ b/tests/find_best_hosts_test.go @@ -2,11 +2,12 @@ package main_test import ( "net" + "os" "reflect" "testing" "time" - "gitlab.cern.ch/lb-experts/golbd/lbcluster" + "lb-experts/golbd/lbcluster" ) func getExpectedHostMetric() map[string]lbcluster.Node { @@ -43,6 +44,11 @@ func TestFindBestHosts(t *testing.T) { if c.Time_of_last_evaluation.Add(time.Duration(2) * time.Second).Before(time.Now()) { t.Errorf("e.Find_best_hosts: c.Time_of_last_evaluation: got\n%v\ncurrent time\n%v", c.Time_of_last_evaluation, time.Now()) } + err = os.Remove("sample.log") + if err != nil { + t.Fail() + t.Errorf("error deleting file.error %v", err) + } } func TestFindBestHostsNoValidHostCmsfrontier(t *testing.T) { diff --git a/tests/get_list_hosts_test.go b/tests/get_list_hosts_test.go index ed0dde1..faede9c 100644 --- a/tests/get_list_hosts_test.go +++ b/tests/get_list_hosts_test.go @@ -32,8 +32,14 @@ func TestGetListHostsOne(t *testing.T) { } hostsToCheck := c.GetHostList() - if !reflect.DeepEqual(hostsToCheck, expected) { - t.Errorf("e.GetHostList: got\n%v\nexpected\n%v", hostsToCheck, expected) + if len(hostsToCheck) != len(expected) { + t.Errorf("length mismatch. expected :%v, actual:%v", len(expected), len(hostsToCheck)) + } + for hostName, actualHost := range hostsToCheck { + expHost := expected[hostName] + if !reflect.DeepEqual(expHost.GetClusterConfig(), actualHost.GetClusterConfig()) { + t.Errorf("mismatch in cluster config. expected:%v,actual:%v", expHost.GetClusterConfig(), actualHost.GetClusterConfig()) + } } } diff --git a/tests/loadClusters_test.go b/tests/loadClusters_test.go index b534d83..5b36e28 100644 --- a/tests/loadClusters_test.go +++ b/tests/loadClusters_test.go @@ -13,7 +13,7 @@ import ( ) func getTestCluster(name string) lbcluster.LBCluster { - lg, _ := logger.NewLoggerFactory("") + lg, _ := logger.NewLoggerFactory("sample.log") return lbcluster.LBCluster{ ClusterConfig: model.CluserConfig{ @@ -36,7 +36,7 @@ func getTestCluster(name string) lbcluster.LBCluster { } func getSecondTestCluster() lbcluster.LBCluster { - lg, _ := logger.NewLoggerFactory("") + lg, _ := logger.NewLoggerFactory("sample.log") return lbcluster.LBCluster{ ClusterConfig: model.CluserConfig{ @@ -57,7 +57,7 @@ func getSecondTestCluster() lbcluster.LBCluster { Current_index: 0} } func getHostsToCheck(c lbcluster.LBCluster) map[string]lbhost.Host { - lg, _ := logger.NewLoggerFactory("") + lg, _ := logger.NewLoggerFactory("sample.log") host1 := lbhost.NewLBHost(c.ClusterConfig, lg) host1.SetName("lxplus132.cern.ch") host1.SetTransportPayload([]lbhost.LBHostTransportResult{ @@ -96,28 +96,28 @@ func getHostsToCheck(c lbcluster.LBCluster) map[string]lbhost.Host { return hostsToCheck } func getBadHostsToCheck(c lbcluster.LBCluster) map[string]lbhost.Host { - lg, _ := logger.NewLoggerFactory("") + lg, _ := logger.NewLoggerFactory("sample.log") host1 := lbhost.NewLBHost(c.ClusterConfig, lg) host1.SetName("lxplus132.cern.ch") host1.SetTransportPayload([]lbhost.LBHostTransportResult{ - lbhost.LBHostTransportResult{Transport: "udp6", Response_int: 2, Response_string: "", IP: net.ParseIP("2001:1458:d00:2c::100:a6"), Response_error: ""}, - lbhost.LBHostTransportResult{Transport: "udp", Response_int: 2, Response_string: "", IP: net.ParseIP("188.184.108.98"), Response_error: ""}, + lbhost.LBHostTransportResult{Transport: "udp6", Response_int: -2, Response_string: "", IP: net.ParseIP("2001:1458:d00:2c::100:a6"), Response_error: ""}, + lbhost.LBHostTransportResult{Transport: "udp", Response_int: -2, Response_string: "", IP: net.ParseIP("188.184.108.98"), Response_error: ""}, }) host2 := lbhost.NewLBHost(c.ClusterConfig, lg) host2.SetName("lxplus041.cern.ch") host2.SetTransportPayload([]lbhost.LBHostTransportResult{ - lbhost.LBHostTransportResult{Transport: "udp6", Response_int: 3, Response_string: "", IP: net.ParseIP("2001:1458:d00:32::100:51"), Response_error: ""}, - lbhost.LBHostTransportResult{Transport: "udp", Response_int: 3, Response_string: "", IP: net.ParseIP("188.184.116.81"), Response_error: ""}, + lbhost.LBHostTransportResult{Transport: "udp6", Response_int: -3, Response_string: "", IP: net.ParseIP("2001:1458:d00:32::100:51"), Response_error: ""}, + lbhost.LBHostTransportResult{Transport: "udp", Response_int: -3, Response_string: "", IP: net.ParseIP("188.184.116.81"), Response_error: ""}, }) host3 := lbhost.NewLBHost(c.ClusterConfig, lg) host3.SetName("lxplus130.cern.ch") host3.SetTransportPayload([]lbhost.LBHostTransportResult{lbhost.LBHostTransportResult{ - Transport: "udp", Response_int: 27, Response_string: "", IP: net.ParseIP("188.184.108.100"), Response_error: "", + Transport: "udp", Response_int: -27, Response_string: "", IP: net.ParseIP("188.184.108.100"), Response_error: "", }}) host4 := lbhost.NewLBHost(c.ClusterConfig, lg) host4.SetName("lxplus133.subdo.cern.ch") host4.SetTransportPayload([]lbhost.LBHostTransportResult{lbhost.LBHostTransportResult{ - Transport: "udp", Response_int: 27, Response_string: "", IP: net.ParseIP("188.184.108.101"), Response_error: "", + Transport: "udp", Response_int: -15, Response_string: "", IP: net.ParseIP("188.184.108.101"), Response_error: "", }}) host5 := lbhost.NewLBHost(c.ClusterConfig, lg) host5.SetName("monit-kafkax-17be060b0d.cern.ch") @@ -135,7 +135,7 @@ func getBadHostsToCheck(c lbcluster.LBCluster) map[string]lbhost.Host { return badHostsToCheck } func getHost(hostname string, responseInt int, responseString string) lbhost.Host { - lg, _ := logger.NewLoggerFactory("") + lg, _ := logger.NewLoggerFactory("sample.log") clusterConfig := model.CluserConfig{ Cluster_name: "test01.cern.ch", Loadbalancing_username: "loadbalancing", @@ -194,5 +194,9 @@ func TestLoadClusters(t *testing.T) { t.Errorf("loadClusters: wrong number of clusters, got\n%v\nexpected\n%v (and %v", len(lbclusters), len(expected), i) } - os.Remove("sample.log") + err := os.Remove("sample.log") + if err != nil { + t.Fail() + t.Errorf("error deleting file.error %v", err) + } } diff --git a/tests/sample.log b/tests/sample.log new file mode 100644 index 0000000..20f4b41 --- /dev/null +++ b/tests/sample.log @@ -0,0 +1,22 @@ +May 14 15:09:59.614 lbd[97034]: INFO: The ips for this host are [] +May 14 15:09:59.614 lbd[97034]: INFO: The ips for this host are [188.184.108.100] +May 14 15:09:59.614 lbd[97034]: INFO: The ips for this host are [] +May 14 15:09:59.614 lbd[97034]: INFO: The ips for this host are [] +May 14 15:09:59.614 lbd[97034]: INFO: The ips for this host are [] +May 14 15:09:59.615 lbd[97034]: INFO: Got metric = cmsfrontier +May 14 15:09:59.615 lbd[97034]: WARNING: no usable hosts found for cluster! Skipping the DNS update +May 14 15:10:22.159 lbd[97076]: INFO: The ips for this host are [] +May 14 15:10:22.160 lbd[97076]: INFO: The ips for this host are [] +May 14 15:10:22.160 lbd[97076]: INFO: The ips for this host are [188.184.108.100] +May 14 15:10:22.160 lbd[97076]: INFO: The ips for this host are [] +May 14 15:10:22.160 lbd[97076]: INFO: The ips for this host are [] +May 14 15:10:22.161 lbd[97076]: INFO: Got metric = cmsfrontier +May 14 15:10:22.161 lbd[97076]: WARNING: no usable hosts found for cluster! Skipping the DNS update +May 14 15:10:35.875 lbd[97115]: INFO: The ips for this host are [] +May 14 15:10:35.875 lbd[97115]: INFO: The ips for this host are [] +May 14 15:10:35.876 lbd[97115]: INFO: The ips for this host are [] +May 14 15:10:35.876 lbd[97115]: INFO: The ips for this host are [188.184.108.100] +May 14 15:10:35.876 lbd[97115]: INFO: The ips for this host are [] +May 14 15:10:35.876 lbd[97115]: INFO: Got metric = minino +May 14 15:10:35.876 lbd[97115]: WARNING: no usable hosts found for cluster! Returning no hosts. +May 14 15:10:35.877 lbd[97115]: INFO: best hosts are: NONE From 8d5c9d0624a3ff8d889f4f02d2a7d4491d7abc7e Mon Sep 17 00:00:00 2001 From: Yadu Nandan Date: Sat, 14 May 2022 17:02:25 -0700 Subject: [PATCH 15/20] mend --- lbcluster/lbcluster.go | 7 +++---- lbconfig/config.go | 5 +++-- lbd.go | 4 ++-- tests/get_list_hosts_test.go | 17 +++++++++++------ tests/loadClusters_test.go | 16 +++++++++++----- tests/sample.log | 22 ---------------------- 6 files changed, 30 insertions(+), 41 deletions(-) delete mode 100644 tests/sample.log diff --git a/lbcluster/lbcluster.go b/lbcluster/lbcluster.go index 5a61c16..5830f59 100644 --- a/lbcluster/lbcluster.go +++ b/lbcluster/lbcluster.go @@ -89,20 +89,19 @@ func (lbc *LBCluster) Time_to_refresh() bool { } //GetHostList Get the hosts for an alias -func (lbc *LBCluster) GetHostList() map[string]lbhost.Host { - hostMap := make(map[string]lbhost.Host) +func (lbc *LBCluster) GetHostList(hostMap map[string]lbhost.Host) { + lbc.Slog.Debug("Getting the list of hosts for the alias") for host := range lbc.Host_metric_table { myHost, ok := hostMap[host] if ok { clusterConfig := myHost.GetClusterConfig() - clusterConfig.Cluster_name = clusterConfig.Cluster_name + "," + clusterConfig.Cluster_name + clusterConfig.Cluster_name = clusterConfig.Cluster_name + "," + lbc.ClusterConfig.Cluster_name } else { myHost = lbhost.NewLBHost(lbc.ClusterConfig, lbc.Slog) } hostMap[host] = myHost } - return hostMap } func (lbc *LBCluster) concatenateNodes(myNodes []Node) string { diff --git a/lbconfig/config.go b/lbconfig/config.go index 6881e1b..e7ed2f1 100644 --- a/lbconfig/config.go +++ b/lbconfig/config.go @@ -5,8 +5,6 @@ import ( "encoding/json" "fmt" "io" - "lb-experts/golbd/logger" - "lb-experts/golbd/model" "net" "os" "strconv" @@ -15,6 +13,8 @@ import ( "time" "lb-experts/golbd/lbcluster" + "lb-experts/golbd/logger" + "lb-experts/golbd/model" ) const ( @@ -35,6 +35,7 @@ type Config interface { UnlockHeartBeatMutex() WatchFileChange(controlChan <-chan bool, waitGroup sync.WaitGroup) <-chan ConfigFileChangeSignal Load() ([]lbcluster.LBCluster, error) + LoadClusters() ([]lbcluster.LBCluster, error) // testing only SetMasterHost(masterHostName string) diff --git a/lbd.go b/lbd.go index 473481e..c71b436 100644 --- a/lbd.go +++ b/lbd.go @@ -214,15 +214,15 @@ func checkAliases(config lbconfig.Config, lg logger.Logger, lbclusters []lbclust //var wg sync.WaitGroup updateDNS := true lg.Info("Checking if any of the " + strconv.Itoa(len(lbclusters)) + " clusters needs updating") - var hostsToCheck map[string]lbhost.Host var clustersToUpdate []*lbcluster.LBCluster + hostsToCheck := make(map[string]lbhost.Host) /* First, let's identify the hosts that have to be checked */ for i := range lbclusters { currentCluster := &lbclusters[i] lg.Debug("DO WE HAVE TO UPDATE?") if currentCluster.Time_to_refresh() { lg.Info("Time to refresh the cluster") - hostsToCheck = currentCluster.GetHostList() + currentCluster.GetHostList(hostsToCheck) clustersToUpdate = append(clustersToUpdate, currentCluster) } } diff --git a/tests/get_list_hosts_test.go b/tests/get_list_hosts_test.go index faede9c..d524673 100644 --- a/tests/get_list_hosts_test.go +++ b/tests/get_list_hosts_test.go @@ -30,8 +30,8 @@ func TestGetListHostsOne(t *testing.T) { "lxplus133.subdo.cern.ch": host4, "lxplus130.cern.ch": host5, } - - hostsToCheck := c.GetHostList() + hostsToCheck := make(map[string]lbhost.Host) + c.GetHostList(hostsToCheck) if len(hostsToCheck) != len(expected) { t.Errorf("length mismatch. expected :%v, actual:%v", len(expected), len(hostsToCheck)) } @@ -41,6 +41,7 @@ func TestGetListHostsOne(t *testing.T) { t.Errorf("mismatch in cluster config. expected:%v,actual:%v", expHost.GetClusterConfig(), actualHost.GetClusterConfig()) } } + os.Remove("sample.log") } func TestGetListHostsTwo(t *testing.T) { @@ -104,12 +105,16 @@ func TestGetListHostsTwo(t *testing.T) { "lxplus025.cern.ch": host4, } - var hostsToCheck map[string]lbhost.Host + hostsToCheck := make(map[string]lbhost.Host) for _, c := range clusters { - hostsToCheck = c.GetHostList() + c.GetHostList(hostsToCheck) } - if !reflect.DeepEqual(hostsToCheck, expected) { - t.Errorf("e.GetHostList: got\n%v\nexpected\n%v", hostsToCheck, expected) + + for hostName, actualHost := range hostsToCheck { + expHost := expected[hostName] + if !reflect.DeepEqual(expHost.GetClusterConfig(), actualHost.GetClusterConfig()) { + t.Errorf("mismatch in cluster config. expected:%v,actual:%v", expHost.GetClusterConfig(), actualHost.GetClusterConfig()) + } } os.Remove("sample.log") } diff --git a/tests/loadClusters_test.go b/tests/loadClusters_test.go index 5b36e28..137c34f 100644 --- a/tests/loadClusters_test.go +++ b/tests/loadClusters_test.go @@ -170,11 +170,17 @@ func TestLoadClusters(t *testing.T) { "test01.cern.ch": lbcluster.Params{Behaviour: "mindless", Best_hosts: 2, External: true, Metric: "cmsfrontier", Polling_interval: 6, Statistics: "long"}, "test02.test.cern.ch": lbcluster.Params{Behaviour: "mindless", Best_hosts: 10, External: false, Metric: "cmsfrontier", Polling_interval: 6, Statistics: "long"}, }) + testCluster1 := getTestCluster("test01.cern.ch") + testCluster1.Slog = lg + testCluster2 := getSecondTestCluster() + testCluster2.Slog = lg + expected := []lbcluster.LBCluster{testCluster1, testCluster2} - expected := []lbcluster.LBCluster{getTestCluster("test01.cern.ch"), - getSecondTestCluster()} - - lbclusters, _ := config.Load() + lbclusters, err := config.LoadClusters() + if err != nil { + t.Errorf("error while loading clusters. error: %v", err) + return + } // reflect.DeepEqual(lbclusters, expected) occassionally fails as the array order is not always the same // so comparing element par element i := 0 @@ -194,7 +200,7 @@ func TestLoadClusters(t *testing.T) { t.Errorf("loadClusters: wrong number of clusters, got\n%v\nexpected\n%v (and %v", len(lbclusters), len(expected), i) } - err := os.Remove("sample.log") + err = os.Remove("sample.log") if err != nil { t.Fail() t.Errorf("error deleting file.error %v", err) diff --git a/tests/sample.log b/tests/sample.log deleted file mode 100644 index 20f4b41..0000000 --- a/tests/sample.log +++ /dev/null @@ -1,22 +0,0 @@ -May 14 15:09:59.614 lbd[97034]: INFO: The ips for this host are [] -May 14 15:09:59.614 lbd[97034]: INFO: The ips for this host are [188.184.108.100] -May 14 15:09:59.614 lbd[97034]: INFO: The ips for this host are [] -May 14 15:09:59.614 lbd[97034]: INFO: The ips for this host are [] -May 14 15:09:59.614 lbd[97034]: INFO: The ips for this host are [] -May 14 15:09:59.615 lbd[97034]: INFO: Got metric = cmsfrontier -May 14 15:09:59.615 lbd[97034]: WARNING: no usable hosts found for cluster! Skipping the DNS update -May 14 15:10:22.159 lbd[97076]: INFO: The ips for this host are [] -May 14 15:10:22.160 lbd[97076]: INFO: The ips for this host are [] -May 14 15:10:22.160 lbd[97076]: INFO: The ips for this host are [188.184.108.100] -May 14 15:10:22.160 lbd[97076]: INFO: The ips for this host are [] -May 14 15:10:22.160 lbd[97076]: INFO: The ips for this host are [] -May 14 15:10:22.161 lbd[97076]: INFO: Got metric = cmsfrontier -May 14 15:10:22.161 lbd[97076]: WARNING: no usable hosts found for cluster! Skipping the DNS update -May 14 15:10:35.875 lbd[97115]: INFO: The ips for this host are [] -May 14 15:10:35.875 lbd[97115]: INFO: The ips for this host are [] -May 14 15:10:35.876 lbd[97115]: INFO: The ips for this host are [] -May 14 15:10:35.876 lbd[97115]: INFO: The ips for this host are [188.184.108.100] -May 14 15:10:35.876 lbd[97115]: INFO: The ips for this host are [] -May 14 15:10:35.876 lbd[97115]: INFO: Got metric = minino -May 14 15:10:35.876 lbd[97115]: WARNING: no usable hosts found for cluster! Returning no hosts. -May 14 15:10:35.877 lbd[97115]: INFO: best hosts are: NONE From 78682b8da961f1485183a7f0402d3d12337f1a0a Mon Sep 17 00:00:00 2001 From: Yadu Nandan Date: Sat, 14 May 2022 18:07:44 -0700 Subject: [PATCH 16/20] mend --- go.mod | 7 ++++++- go.sum | 26 ++++++++++++++++++++++---- 2 files changed, 28 insertions(+), 5 deletions(-) diff --git a/go.mod b/go.mod index 8519926..267294a 100644 --- a/go.mod +++ b/go.mod @@ -3,8 +3,13 @@ module lb-experts/golbd go 1.17 require ( - github.com/miekg/dns v0.0.0-20160605072344-799de7044d95 + github.com/miekg/dns v1.0.0 github.com/reguero/go-snmplib v0.0.0-20181019092238-e566f5619b55 gitlab.cern.ch/lb-experts/golbd v0.2.9 + gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b + golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550 // indirect + golang.org/x/net v0.0.0-20210726213435-c6fcb2dbf985 // indirect + golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c // indirect ) + replace gitlab.cern.ch/lb-experts/golbd => ./ diff --git a/go.sum b/go.sum index 5a33ee4..457f2d3 100644 --- a/go.sum +++ b/go.sum @@ -1,6 +1,24 @@ -github.com/miekg/dns v0.0.0-20160605072344-799de7044d95 h1:BQDvi4qiqpxb1Q2E59rpiT9wTuRNYZAvPMiFp/AUtdg= -github.com/miekg/dns v0.0.0-20160605072344-799de7044d95/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= +github.com/miekg/dns v1.0.0 h1:DZ3fdvcFXfWew8XOY+33+MqAcCnqDrGsnt3kK8yf4Hg= +github.com/miekg/dns v1.0.0/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= github.com/reguero/go-snmplib v0.0.0-20181019092238-e566f5619b55 h1:XSHUgUOYEcULzyllG50slUg8RFmx0CPtr0xoNoft/ng= github.com/reguero/go-snmplib v0.0.0-20181019092238-e566f5619b55/go.mod h1:ZOnUjV/tCr0PrDl0kI9jntec7FWW5prJ1FGD8hZ+I5A= -gitlab.cern.ch/lb-experts/golbd v0.2.9 h1:zzvn2WmUxZL0lCJG7DJgY8sLzqfMzyBHkRR7ppAJKK0= -gitlab.cern.ch/lb-experts/golbd v0.2.9/go.mod h1:8fvn3Xj9jTPFqCeV0jNy7ZJN1ui6xsM5a/cWRywIjBE= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo= +gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550 h1:ObdrDkeb4kJdCP557AjRjq69pTHfNouLtWZG7j9rPN8= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20210726213435-c6fcb2dbf985 h1:4CSI6oo7cOjJKajidEljs9h+uP0rRZBPPPhcCbj5mw8= +golang.org/x/net v0.0.0-20210726213435-c6fcb2dbf985/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c h1:F1jZWGFhYfh0Ci55sIpILtKKK8p3i2/krTr0H1rg74I= +golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= From 0ccebc66d6dd7c925b5c7cc07d908ab45b8036bb Mon Sep 17 00:00:00 2001 From: Yadu Nandan Date: Sat, 14 May 2022 20:23:32 -0700 Subject: [PATCH 17/20] minor correction to log snapshots --- logger/lbcluster_log.go | 8 +++----- tests/lbcluster_log_test.go | 3 +-- 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/logger/lbcluster_log.go b/logger/lbcluster_log.go index 4cfecab..61bf951 100644 --- a/logger/lbcluster_log.go +++ b/logger/lbcluster_log.go @@ -30,6 +30,7 @@ type Log struct { snapShotCycleTime time.Duration logFileBasePath string logFileExtension string + snapshotCounter int } //Logger struct for the Logger interface @@ -108,11 +109,8 @@ func (l *Log) GetLogFilePath() string { } func (l *Log) startSnapShot() { l.logStartTime = time.Now() - l.filePath = fmt.Sprintf("%s_%s.%s", l.logFileBasePath, l.getFileNameTimeSuffix(), l.logFileExtension) -} - -func (l *Log) getFileNameTimeSuffix() string { - return fmt.Sprintf("%v_%v_%v-%v_%v_%v", l.logStartTime.Year(), l.logStartTime.Month(), l.logStartTime.Day(), l.logStartTime.Hour(), l.logStartTime.Minute(), l.logStartTime.Second()) + l.filePath = fmt.Sprintf("%s.%d.%s", l.logFileBasePath, l.snapshotCounter, l.logFileExtension) + l.snapshotCounter += 1 } func (l *Log) shouldStartNewSnapshot() bool { diff --git a/tests/lbcluster_log_test.go b/tests/lbcluster_log_test.go index 330143a..f2f75ac 100644 --- a/tests/lbcluster_log_test.go +++ b/tests/lbcluster_log_test.go @@ -51,8 +51,7 @@ func TestLBClusterLoggerForSnapshot(t *testing.T) { t.Errorf("logger instance is nil") } logger.StartSnapshot(1 * time.Minute) - if !strings.Contains(logger.GetLogFilePath(), - fmt.Sprintf("%v_%v_%v-%v_%v", time.Now().Year(), time.Now().Month(), time.Now().Day(), time.Now().Hour(), time.Now().Minute())) { + if !strings.HasSuffix(logger.GetLogFilePath(), "sample.0.log") { t.Fail() t.Errorf("error while setting snapshot") } From f167a389a1a33986ff9ce1616b6bff5c427820e7 Mon Sep 17 00:00:00 2001 From: Yadu Nandan Date: Sat, 14 May 2022 20:47:10 -0700 Subject: [PATCH 18/20] changes to metrics collector --- lbcluster/lbcluster.go | 2 ++ lbcluster/lbcluster_dns.go | 13 ++++++++++++- lbconfig/config.go | 8 +++++++- lbd.go | 21 ++++++++++++++++----- metric/metric_controller.go | 4 ++-- metric/metric_logic.go | 7 ++++++- 6 files changed, 45 insertions(+), 10 deletions(-) diff --git a/lbcluster/lbcluster.go b/lbcluster/lbcluster.go index 5830f59..619c7a7 100644 --- a/lbcluster/lbcluster.go +++ b/lbcluster/lbcluster.go @@ -4,6 +4,7 @@ import ( "encoding/json" "fmt" "io/ioutil" + "lb-experts/golbd/metric" "math/rand" "net" "net/http" @@ -30,6 +31,7 @@ type LBCluster struct { Previous_best_ips_dns []net.IP Current_index int Slog logger.Logger + MetricLogic metric.Logic } //Params of the alias diff --git a/lbcluster/lbcluster_dns.go b/lbcluster/lbcluster_dns.go index 396e3e1..4f170cb 100644 --- a/lbcluster/lbcluster_dns.go +++ b/lbcluster/lbcluster_dns.go @@ -2,6 +2,7 @@ package lbcluster import ( "fmt" + "lb-experts/golbd/metric" "net" "time" @@ -75,11 +76,21 @@ func (lbc *LBCluster) updateDNS(keyName, tsigKey, dnsManager string) error { c := new(dns.Client) m.SetTsig(keyName, dns.HmacMD5, 300, time.Now().Unix()) c.TsigSecret = map[string]string{keyName: tsigKey} + updateStartTime := time.Now() err = retryModule.Execute(func() error { _, _, err := c.Exchange(m, dnsManager+":53") return err }) - + updateEndTime := time.Now() + if lbc.MetricLogic != nil { + err := lbc.MetricLogic.WriteRecord(metric.Property{ + RoundTripStartTime: updateStartTime, + RoundTripEndTime: updateEndTime, + }) + if err != nil { + return err + } + } if err != nil { lbc.Slog.Error(fmt.Sprintf("DNS update failed with (%v)", err)) return err diff --git a/lbconfig/config.go b/lbconfig/config.go index e4bebb3..c100823 100644 --- a/lbconfig/config.go +++ b/lbconfig/config.go @@ -5,6 +5,7 @@ import ( "encoding/json" "fmt" "io" + "lb-experts/golbd/metric" "net" "os" "strconv" @@ -341,7 +342,11 @@ func readLines(path string) (lines []string, err error) { func (c *LBConfig) LoadClusters() ([]lbcluster.LBCluster, error) { var lbc lbcluster.LBCluster var lbcs []lbcluster.LBCluster - + hostName, err := os.Hostname() + if err != nil { + return nil, err + } + metricLogic := metric.NewLogic(DefaultMetricsDirectoryPath, hostName) for k, v := range c.Clusters { if len(v) == 0 { c.lbLog.Warning("cluster: " + k + " ignored as it has no members defined in the configuration file " + c.configFilePath) @@ -359,6 +364,7 @@ func (c *LBConfig) LoadClusters() ([]lbcluster.LBCluster, error) { Current_best_ips: []net.IP{}, Previous_best_ips_dns: []net.IP{}, Slog: c.lbLog, + MetricLogic: metricLogic, } hm := make(map[string]lbcluster.Node) for _, h := range v { diff --git a/lbd.go b/lbd.go index 941acbc..7c6d98a 100644 --- a/lbd.go +++ b/lbd.go @@ -4,6 +4,7 @@ import ( "flag" "fmt" "io/ioutil" + "lb-experts/golbd/metric" "log" "math/rand" "os" @@ -38,11 +39,12 @@ var ( ) const ( - itCSgroupDNSserver string = "cfmgr.cern.ch" - DefaultSleepDuration = 10 - DefaultLbdTag = "lbd" - DefaultConnectionTimeout = 10 * time.Second - DefaultReadTimeout = 20 * time.Second + shouldStartMetricServer = false // server disabled by default + itCSgroupDNSserver = "cfmgr.cern.ch" + DefaultSleepDuration = 10 + DefaultLbdTag = "lbd" + DefaultConnectionTimeout = 10 * time.Second + DefaultReadTimeout = 20 * time.Second ) type ConfigFileChangeSignal struct { @@ -182,6 +184,15 @@ func main() { fileChangeSignal := lbConfig.WatchFileChange(controlChan, wg) intervalTickerSignal := sleep(DefaultSleepDuration, controlChan, wg) + if shouldStartMetricServer { + go func() { + err := metric.NewMetricServer(lbconfig.DefaultMetricsDirectoryPath) + if err != nil { + logger.Error(fmt.Sprintf("error while starting metric server . error: %v", err)) + } + }() + } + for { select { case fileWatcherData := <-fileChangeSignal: diff --git a/metric/metric_controller.go b/metric/metric_controller.go index 827c777..39254d2 100644 --- a/metric/metric_controller.go +++ b/metric/metric_controller.go @@ -11,10 +11,10 @@ import ( const defaultPort = ":8081" type serverCtx struct { - metricLogic *BizLogic + metricLogic Logic } -func StartNewMetricServer(metricDirectoryPath string) error { +func NewMetricServer(metricDirectoryPath string) error { currentHostname, err := os.Hostname() if err != nil { return fmt.Errorf("unable to fetch current host name. error: %v", err) diff --git a/metric/metric_logic.go b/metric/metric_logic.go index c459ca3..8a1fca3 100644 --- a/metric/metric_logic.go +++ b/metric/metric_logic.go @@ -15,6 +15,11 @@ const ( defaultCycleDuration = 24 * time.Hour ) +type Logic interface { + GetFilePath() string + ReadHostMetric() (HostMetric, error) + WriteRecord(property Property) error +} type BizLogic struct { isCurrentHostMaster bool hostName string @@ -24,7 +29,7 @@ type BizLogic struct { recordStartTime time.Time } -func NewLogic(dirPath string, hostName string) *BizLogic { +func NewLogic(dirPath string, hostName string) Logic { logic := &BizLogic{ dirPath: dirPath, hostName: hostName, From 45e37b99bcb95c22695b42b60b86938c611516bd Mon Sep 17 00:00:00 2001 From: Yadu Nandan Date: Sat, 14 May 2022 21:00:08 -0700 Subject: [PATCH 19/20] minor fix to evalhosts --- lbcluster/lbcluster.go | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/lbcluster/lbcluster.go b/lbcluster/lbcluster.go index 619c7a7..f9b9c6e 100644 --- a/lbcluster/lbcluster.go +++ b/lbcluster/lbcluster.go @@ -295,13 +295,20 @@ func (lbc *LBCluster) EvaluateHosts(hostsToCheck map[string]lbhost.Host) { var nodeChan = make(chan Node) defer close(nodeChan) var wg sync.WaitGroup - for currentHost := range lbc.Host_metric_table { + newHostMetricMap := make(map[string]Node) + for k, v := range lbc.Host_metric_table { + newHostMetricMap[k] = v + } + for currentHost := range newHostMetricMap { wg.Add(1) go func(selectedHost string) { host := hostsToCheck[selectedHost] ips, err := host.GetWorkingIPs() if err != nil { ips, err = host.GetIps() + if err != nil { + lbc.Slog.Error(fmt.Sprintf("error while fetching IPs. error: %v", err)) + } } nodeChan <- Node{host.GetLoadForAlias(lbc.ClusterConfig.Cluster_name), ips, selectedHost} }(currentHost) From 9e295d5eba2522a284706da7ff928f9752778090 Mon Sep 17 00:00:00 2001 From: Yadu Nandan Date: Sun, 12 Jun 2022 18:14:02 -0700 Subject: [PATCH 20/20] minor fixes and refractors --- lbcluster/lbcluster.go | 2 +- lbcluster/retry_module.go | 5 +++-- lbconfig/config.go | 2 +- lbd.go | 5 ----- lbhost/lbhost.go | 10 +++++----- model/cluster_config.go | 2 +- tests/evaluate_hosts_internal_test.go | 2 +- tests/get_list_hosts_test.go | 12 ++++++------ tests/get_state_dns_test.go | 2 +- tests/loadClusters_test.go | 6 +++--- 10 files changed, 22 insertions(+), 26 deletions(-) diff --git a/lbcluster/lbcluster.go b/lbcluster/lbcluster.go index f9b9c6e..717ba56 100644 --- a/lbcluster/lbcluster.go +++ b/lbcluster/lbcluster.go @@ -23,7 +23,7 @@ const WorstValue int = 99999 //LBCluster struct of an lbcluster alias type LBCluster struct { - ClusterConfig model.CluserConfig + ClusterConfig model.ClusterConfig Host_metric_table map[string]Node Parameters Params Time_of_last_evaluation time.Time diff --git a/lbcluster/retry_module.go b/lbcluster/retry_module.go index ae962be..1e22bb4 100644 --- a/lbcluster/retry_module.go +++ b/lbcluster/retry_module.go @@ -35,7 +35,7 @@ func NewRetryModule(retryStartDuration time.Duration, logger logger.Logger) *Ret func (r *Retry) SetMaxDuration(maxDuration time.Duration) error { if r.retryStarted { - return nil + return fmt.Errorf("retry routine has already started") } if maxDuration <= 0 { return fmt.Errorf("duration has to be greater than 0") @@ -46,7 +46,7 @@ func (r *Retry) SetMaxDuration(maxDuration time.Duration) error { func (r *Retry) SetMaxCount(maxCount int) error { if r.retryStarted { - return nil + return fmt.Errorf("retry routine has already started") } if maxCount <= 0 { return fmt.Errorf("max count has to be greater than 0") @@ -85,6 +85,7 @@ func (r *Retry) run() { ticker := time.NewTicker(1 * time.Minute) defer close(r.done) defer close(r.signal) + defer close(start) defer ticker.Stop() for { diff --git a/lbconfig/config.go b/lbconfig/config.go index c100823..1ce91ee 100644 --- a/lbconfig/config.go +++ b/lbconfig/config.go @@ -353,7 +353,7 @@ func (c *LBConfig) LoadClusters() ([]lbcluster.LBCluster, error) { continue } if par, ok := c.Parameters[k]; ok { - lbcConfig := model.CluserConfig{ + lbcConfig := model.ClusterConfig{ Cluster_name: k, Loadbalancing_username: DefaultLoadBalancerConfig, Loadbalancing_password: c.SnmpPassword, diff --git a/lbd.go b/lbd.go index 7c6d98a..ca38898 100644 --- a/lbd.go +++ b/lbd.go @@ -47,11 +47,6 @@ const ( DefaultReadTimeout = 20 * time.Second ) -type ConfigFileChangeSignal struct { - readSignal bool - readError error -} - func shouldUpdateDNS(config lbconfig.Config, hostname string, lg logger.Logger) bool { if strings.EqualFold(hostname, config.GetMasterHost()) { return true diff --git a/lbhost/lbhost.go b/lbhost/lbhost.go index 54914e1..37e6c61 100644 --- a/lbhost/lbhost.go +++ b/lbhost/lbhost.go @@ -27,7 +27,7 @@ type LBHostTransportResult struct { Response_error string } type LBHost struct { - ClusterConfig model.CluserConfig + ClusterConfig model.ClusterConfig Host_name string HostTransports []LBHostTransportResult Logger logger.Logger @@ -45,7 +45,7 @@ type DiscoveryAgent interface { GetV3(oid snmplib.Oid) (interface{}, error) } -func NewHostDiscoveryAgent(nodeIp string, clusterConfig model.CluserConfig) (DiscoveryAgent, error) { +func NewHostDiscoveryAgent(nodeIp string, clusterConfig model.ClusterConfig) (DiscoveryAgent, error) { return snmplib.NewSNMPv3(nodeIp, clusterConfig.Loadbalancing_username, "MD5", clusterConfig.Loadbalancing_password, "NOPRIV", clusterConfig.Loadbalancing_password, time.Duration(TIMEOUT)*time.Second, 2) @@ -55,7 +55,7 @@ type Host interface { GetName() string SetName(name string) SNMPDiscovery() - GetClusterConfig() *model.CluserConfig + GetClusterConfig() *model.ClusterConfig GetLoadForAlias(clusterName string) int GetWorkingIPs() ([]net.IP, error) GetAllIPs() ([]net.IP, error) @@ -64,7 +64,7 @@ type Host interface { GetHostTransportPayloads() []LBHostTransportResult } -func NewLBHost(clusterConfig model.CluserConfig, logger logger.Logger) Host { +func NewLBHost(clusterConfig model.ClusterConfig, logger logger.Logger) Host { return &LBHost{ ClusterConfig: clusterConfig, Logger: logger, @@ -78,7 +78,7 @@ func (lh *LBHost) SetName(name string) { func (lh *LBHost) GetName() string { return lh.Host_name } -func (lh *LBHost) GetClusterConfig() *model.CluserConfig { +func (lh *LBHost) GetClusterConfig() *model.ClusterConfig { return &lh.ClusterConfig } diff --git a/model/cluster_config.go b/model/cluster_config.go index 14038bb..3b3e2f0 100644 --- a/model/cluster_config.go +++ b/model/cluster_config.go @@ -1,6 +1,6 @@ package model -type CluserConfig struct { +type ClusterConfig struct { Cluster_name string Loadbalancing_username string Loadbalancing_password string diff --git a/tests/evaluate_hosts_internal_test.go b/tests/evaluate_hosts_internal_test.go index 40b8f7f..7185e9c 100644 --- a/tests/evaluate_hosts_internal_test.go +++ b/tests/evaluate_hosts_internal_test.go @@ -36,7 +36,7 @@ func (m mockHost) SNMPDiscovery() { panic("implement me") } -func (m mockHost) GetClusterConfig() *model.CluserConfig { +func (m mockHost) GetClusterConfig() *model.ClusterConfig { panic("implement me") } diff --git a/tests/get_list_hosts_test.go b/tests/get_list_hosts_test.go index d524673..f364b95 100644 --- a/tests/get_list_hosts_test.go +++ b/tests/get_list_hosts_test.go @@ -49,7 +49,7 @@ func TestGetListHostsTwo(t *testing.T) { logger.EnableWriteToSTd() clusters := []lbcluster.LBCluster{ - {ClusterConfig: model.CluserConfig{ + {ClusterConfig: model.ClusterConfig{ Cluster_name: "test01.cern.ch", Loadbalancing_username: "loadbalancing", Loadbalancing_password: "zzz123", @@ -61,7 +61,7 @@ func TestGetListHostsTwo(t *testing.T) { Previous_best_ips_dns: []net.IP{}, Slog: logger, Current_index: 0}, - lbcluster.LBCluster{ClusterConfig: model.CluserConfig{ + lbcluster.LBCluster{ClusterConfig: model.ClusterConfig{ Cluster_name: "test02.cern.ch", Loadbalancing_username: "loadbalancing", Loadbalancing_password: "zzz123", @@ -74,25 +74,25 @@ func TestGetListHostsTwo(t *testing.T) { Slog: logger, Current_index: 0}} - host1 := lbhost.NewLBHost(model.CluserConfig{ + host1 := lbhost.NewLBHost(model.ClusterConfig{ Cluster_name: "test01.cern.ch", Loadbalancing_username: "loadbalancing", Loadbalancing_password: "zzz123", }, logger) host1.SetName("lxplus142.cern.ch") - host2 := lbhost.NewLBHost(model.CluserConfig{ + host2 := lbhost.NewLBHost(model.ClusterConfig{ Cluster_name: "test01.cern.ch,test02.cern.ch", Loadbalancing_username: "loadbalancing", Loadbalancing_password: "zzz123", }, logger) host2.SetName("lxplus177.cern.ch") - host3 := lbhost.NewLBHost(model.CluserConfig{ + host3 := lbhost.NewLBHost(model.ClusterConfig{ Cluster_name: "test02.cern.ch", Loadbalancing_username: "loadbalancing", Loadbalancing_password: "zzz123", }, logger) host3.SetName("lxplus013.cern.ch") - host4 := lbhost.NewLBHost(model.CluserConfig{ + host4 := lbhost.NewLBHost(model.ClusterConfig{ Cluster_name: "test02.cern.ch", Loadbalancing_username: "loadbalancing", Loadbalancing_password: "zzz123", diff --git a/tests/get_state_dns_test.go b/tests/get_state_dns_test.go index 1539692..c8493d3 100644 --- a/tests/get_state_dns_test.go +++ b/tests/get_state_dns_test.go @@ -84,7 +84,7 @@ func TestRefreshDNS(t *testing.T) { t.Run(tc.cluster_name, func(t *testing.T) { lg, _ := logger.NewLoggerFactory("sample.log") cluster := lbcluster.LBCluster{ - ClusterConfig: model.CluserConfig{ + ClusterConfig: model.ClusterConfig{ Cluster_name: tc.cluster_name, }, Current_best_ips: tc.current_best_ips, diff --git a/tests/loadClusters_test.go b/tests/loadClusters_test.go index 52907ee..57fab5c 100644 --- a/tests/loadClusters_test.go +++ b/tests/loadClusters_test.go @@ -16,7 +16,7 @@ func getTestCluster(name string) lbcluster.LBCluster { lg, _ := logger.NewLoggerFactory("sample.log") return lbcluster.LBCluster{ - ClusterConfig: model.CluserConfig{ + ClusterConfig: model.ClusterConfig{ Cluster_name: name, Loadbalancing_username: "loadbalancing", Loadbalancing_password: "zzz123", @@ -39,7 +39,7 @@ func getSecondTestCluster() lbcluster.LBCluster { lg, _ := logger.NewLoggerFactory("sample.log") return lbcluster.LBCluster{ - ClusterConfig: model.CluserConfig{ + ClusterConfig: model.ClusterConfig{ Cluster_name: "test02.test.cern.ch", Loadbalancing_username: "loadbalancing", Loadbalancing_password: "zzz123", @@ -136,7 +136,7 @@ func getBadHostsToCheck(c lbcluster.LBCluster) map[string]lbhost.Host { } func getHost(hostname string, responseInt int, responseString string) lbhost.Host { lg, _ := logger.NewLoggerFactory("sample.log") - clusterConfig := model.CluserConfig{ + clusterConfig := model.ClusterConfig{ Cluster_name: "test01.cern.ch", Loadbalancing_username: "loadbalancing", Loadbalancing_password: "XXXX",