diff --git a/cmd/attack/attack.go b/cmd/attack/attack.go index b7ed71a2..eca0be49 100644 --- a/cmd/attack/attack.go +++ b/cmd/attack/attack.go @@ -42,6 +42,7 @@ func NewAttackCommand() *cobra.Command { NewHTTPAttackCommand(&uid), NewVMAttackCommand(&uid), NewUserDefinedCommand(&uid), + NewPatroniAttackCommand(&uid), ) return cmd diff --git a/cmd/attack/patroni.go b/cmd/attack/patroni.go index 84260994..bb7849dd 100644 --- a/cmd/attack/patroni.go +++ b/cmd/attack/patroni.go @@ -48,6 +48,9 @@ func NewPatroniAttackCommand(uid *string) *cobra.Command { cmd.PersistentFlags().StringVarP(&options.User, "user", "u", "patroni", "patroni cluster user") cmd.PersistentFlags().StringVar(&options.Password, "password", "p", "patroni cluster password") + cmd.PersistentFlags().StringVarP(&options.Address, "address", "a", "", "patroni cluster address, any of available hosts") + cmd.PersistentFlags().BoolVarP(&options.LocalMode, "local-mode", "l", false, "execute patronictl on host with chaosd. User with privileges required.") + cmd.PersistentFlags().BoolVarP(&options.RemoteMode, "remote-mode", "r", false, "execute patroni command by REST API") return cmd } @@ -55,15 +58,15 @@ func NewPatroniAttackCommand(uid *string) *cobra.Command { func NewPatroniSwitchoverCommand(dep fx.Option, options *core.PatroniCommand) *cobra.Command { cmd := &cobra.Command{ Use: "switchover", - Short: "exec switchover, default without another attack. Warning! Command is not recover!", + Short: "exec switchover command. Warning! Command is not recover!", Run: func(*cobra.Command, []string) { options.Action = core.SwitchoverAction utils.FxNewAppWithoutLog(dep, fx.Invoke(PatroniAttackF)).Run() }, } - cmd.Flags().StringVarP(&options.Address, "address", "a", "", "patroni cluster address, any of available hosts") cmd.Flags().StringVarP(&options.Candidate, "candidate", "c", "", "switchover candidate, default random unit for replicas") - cmd.Flags().StringVarP(&options.Scheduled_at, "scheduled_at", "d", fmt.Sprintln(time.Now().Add(time.Second*60).Format(time.RFC3339)), "scheduled switchover, default now()+1 minute") + cmd.Flags().StringVarP(&options.Scheduled_at, "scheduled_at", "d", fmt.Sprint(time.Now().Add(time.Second*60).Format(time.RFC3339)), `scheduled switchover, + default now()+1 minute by remote mode`) return cmd } @@ -71,14 +74,13 @@ func NewPatroniSwitchoverCommand(dep fx.Option, options *core.PatroniCommand) *c func NewPatroniFailoverCommand(dep fx.Option, options *core.PatroniCommand) *cobra.Command { cmd := &cobra.Command{ Use: "failover", - Short: "exec failover, default without another attack", + Short: "Exec failover command. Warning! Command is not recover!", Run: func(*cobra.Command, []string) { options.Action = core.FailoverAction utils.FxNewAppWithoutLog(dep, fx.Invoke(PatroniAttackF)).Run() }, } - cmd.Flags().StringVarP(&options.Address, "address", "a", "", "patroni cluster address, any of available hosts") cmd.Flags().StringVarP(&options.Candidate, "leader", "c", "", "failover new leader, default random unit for replicas") return cmd } diff --git a/pkg/core/disk_test.go b/pkg/core/disk_test.go index a57e42e9..e46d7299 100644 --- a/pkg/core/disk_test.go +++ b/pkg/core/disk_test.go @@ -4,12 +4,13 @@ // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // -// http://www.apache.org/licenses/LICENSE-2.0 +// http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // See the License for the specific language governing permissions and // limitations under the License. + package core import ( diff --git a/pkg/core/jvm.go b/pkg/core/jvm.go index c28752bc..38ec4dd4 100644 --- a/pkg/core/jvm.go +++ b/pkg/core/jvm.go @@ -141,8 +141,8 @@ type JVMStressSpec struct { // only when SQL match the Database, Table and SQLType, chaosd will inject fault // for example: // -// SQL is "select * from test.t1", -// only when ((Database == "test" || Database == "") && (Table == "t1" || Table == "") && (SQLType == "select" || SQLType == "")) is true, chaosd will inject fault +// SQL is "select * from test.t1", +// only when ((Database == "test" || Database == "") && (Table == "t1" || Table == "") && (SQLType == "select" || SQLType == "")) is true, chaosd will inject fault type JVMMySQLSpec struct { // the version of mysql-connector-java, only support 5.X.X(set to 5) and 8.X.X(set to 8) now MySQLConnectorVersion string `json:"mysql-connector-version,omitempty"` diff --git a/pkg/core/patroni.go b/pkg/core/patroni.go index 5305a491..bed91264 100644 --- a/pkg/core/patroni.go +++ b/pkg/core/patroni.go @@ -35,6 +35,8 @@ type PatroniCommand struct { User string `json:"user,omitempty"` Password string `json:"password,omitempty"` Scheduled_at string `json:"scheduled_at,omitempty"` + LocalMode bool `json:"local_mode,omitempty"` + RemoteMode bool `json:"remote_mode,omitempty"` RecoverCmd string `json:"recoverCmd,omitempty"` } @@ -42,16 +44,24 @@ func (p *PatroniCommand) Validate() error { if err := p.CommonAttackConfig.Validate(); err != nil { return err } - if len(p.Address) == 0 { - return errors.New("address not provided") - } - if len(p.User) == 0 { - return errors.New("patroni user not provided") + if !p.RemoteMode && !p.LocalMode { + return errors.New("local or remote mode required") } - if len(p.Password) == 0 { - return errors.New("patroni password not provided") + if p.RemoteMode { + + if len(p.Address) == 0 { + return errors.New("address not provided") + } + + if len(p.User) == 0 { + return errors.New("patroni user not provided") + } + + if len(p.Password) == 0 { + return errors.New("patroni password not provided") + } } return nil diff --git a/pkg/server/chaosd/disk_test.go b/pkg/server/chaosd/disk_test.go index 7aa407d8..d66577b6 100644 --- a/pkg/server/chaosd/disk_test.go +++ b/pkg/server/chaosd/disk_test.go @@ -4,12 +4,13 @@ // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // -// http://www.apache.org/licenses/LICENSE-2.0 +// http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // See the License for the specific language governing permissions and // limitations under the License. + package chaosd import ( diff --git a/pkg/server/chaosd/patroni.go b/pkg/server/chaosd/patroni.go index 040f957f..3090387f 100644 --- a/pkg/server/chaosd/patroni.go +++ b/pkg/server/chaosd/patroni.go @@ -14,17 +14,18 @@ package chaosd import ( - "bytes" "encoding/json" "fmt" - "io" "math/rand" "net/http" + "os/exec" + "strings" - "github.com/chaos-mesh/chaosd/pkg/core" - "github.com/chaos-mesh/chaosd/pkg/server/utils" "github.com/pingcap/errors" "github.com/pingcap/log" + + "github.com/chaos-mesh/chaosd/pkg/core" + "github.com/chaos-mesh/chaosd/pkg/server/utils" ) type patroniAttack struct{} @@ -34,17 +35,24 @@ var PatroniAttack AttackType = patroniAttack{} func (patroniAttack) Attack(options core.AttackConfig, _ Environment) error { attack := options.(*core.PatroniCommand) - candidate := attack.Candidate - - leader := attack.Leader + var responce []byte - var scheduled_at string + var address string - var url string + var err error values := make(map[string]string) - patroniInfo, err := utils.GetPatroniInfo(attack.Address) + if attack.RemoteMode { + address = attack.Address + } else if attack.LocalMode { + address, err = utils.GetLocalHostname() + if err != nil { + return errors.WithStack(err) + } + } + + patroniInfo, err := utils.GetPatroniInfo(address) if err != nil { err = errors.Errorf("failed to get patroni info for %v: %v", options.String(), err) return errors.WithStack(err) @@ -55,75 +63,129 @@ func (patroniAttack) Attack(options core.AttackConfig, _ Environment) error { return errors.WithStack(err) } - if candidate == "" { - candidate = patroniInfo.Replicas[rand.Intn(len(patroniInfo.Replicas))] + sync_mode_check, err := isSynchronousClusterMode(address, attack.User, attack.Password) + if err != nil { + err = errors.Errorf("failed to check cluster synchronous mode for %v: %v", options.String(), err) + return errors.WithStack(err) + } + + if attack.Candidate == "" { + if sync_mode_check { + values["candidate"] = patroniInfo.SyncStandby[rand.Intn(len(patroniInfo.SyncStandby))] + } else { + values["candidate"] = patroniInfo.Replicas[rand.Intn(len(patroniInfo.Replicas))] + } + } - if leader == "" { - leader = patroniInfo.Master + if attack.Leader == "" { + values["leader"] = patroniInfo.Master } - switch options.String() { - case "switchover": + values["scheduled_at"] = attack.Scheduled_at - scheduled_at = attack.Scheduled_at + cmd := options.String() - values = map[string]string{"leader": leader, "scheduled_at": scheduled_at} + switch cmd { + case "switchover": - log.Info(fmt.Sprintf("Switchover will be done from %v to another available replica in %v", patroniInfo.Master, scheduled_at)) + log.Info(fmt.Sprintf("Switchover will be done from %v to %v in %v", values["leader"], values["candidate"], values["scheduled_at"])) case "failover": - values = map[string]string{"candidate": candidate} + log.Info(fmt.Sprintf("Failover will be done from %v to %v", values["leader"], values["candidate"])) - log.Info(fmt.Sprintf("Failover will be done from %v to %v", patroniInfo.Master, candidate)) + } + if attack.RemoteMode { + responce, err = execPatroniAttackByRemoteMode(address, attack.User, attack.Password, cmd, values) + if err != nil { + return err + } + } else if attack.LocalMode { + responce, err = execPatroniAttackByLocalMode(cmd, values) + if err != nil { + return err + } } - patroniAddr := attack.Address + if attack.RemoteMode { + log.S().Infof("Execute %v successfully: %v", cmd, string(responce)) + } - cmd := options.String() + if attack.LocalMode { + log.S().Infof("Execute %v successfully", cmd) + fmt.Println(string(responce)) + } + + return nil +} + +func execPatroniAttackByRemoteMode(patroniAddr string, user string, password string, cmd string, values map[string]string) ([]byte, error) { data, err := json.Marshal(values) if err != nil { err = errors.Errorf("failed to marshal data: %v", values) - return errors.WithStack(err) + return nil, errors.WithStack(err) } - url = fmt.Sprintf("http://%v:8008/%v", patroniAddr, cmd) + buf, err := utils.MakeHTTPRequest(http.MethodPost, patroniAddr, 8008, cmd, data, user, password) + if err != nil { + return nil, errors.WithStack(err) + } - request, err := http.NewRequest("POST", url, bytes.NewBuffer(data)) + return buf, nil +} + +func execPatroniAttackByLocalMode(cmd string, values map[string]string) ([]byte, error) { + var cmdTemplate string + + if cmd == "failover" { + cmdTemplate = fmt.Sprintf("patronictl %v --master %v --candidate %v --force", cmd, values["leader"], values["candidate"]) + } else if cmd == "switchover" { + cmdTemplate = fmt.Sprintf("patronictl %v --master %v --candidate %v --scheduled %v --force", cmd, values["leader"], values["candidate"], values["scheduled_at"]) + } + + execCmd := exec.Command("bash", "-c", cmdTemplate) + output, err := execCmd.CombinedOutput() if err != nil { - err = errors.Errorf("failed to %v: %v", cmd, err) - return errors.WithStack(err) + log.S().Errorf(fmt.Sprintf("failed to %v: %v", cmdTemplate, string(output))) + return nil, err } - request.Header.Set("Content-Type", "application/json") - request.SetBasicAuth(attack.User, attack.Password) + if strings.Contains(string(output), "failed") { + err = errors.New(string(output)) + return nil, err + } - client := &http.Client{} - resp, error := client.Do(request) - if error != nil { - err = errors.Errorf("failed to %v: %v", cmd, err) - return errors.WithStack(err) + return output, nil +} + +func isSynchronousClusterMode(patroniAddr string, user string, password string) (bool, error) { + + buf, err := utils.MakeHTTPRequest(http.MethodGet, patroniAddr, 8008, "config", []byte{}, user, password) + if err != nil { + return false, err } - defer resp.Body.Close() + patroni_responce := make(map[string]interface{}) - buf, err := io.ReadAll(resp.Body) + err = json.Unmarshal(buf, &patroni_responce) if err != nil { - err = errors.Errorf("failed to read %v responce: %v", cmd, err) - return errors.WithStack(err) + return false, fmt.Errorf("bad request %v %v", err.Error(), http.StatusBadRequest) } - if resp.StatusCode != 200 && resp.StatusCode != 202 { - err = errors.Errorf("failed to %v: status code %v, responce %v", cmd, resp.StatusCode, string(buf)) - return errors.WithStack(err) + mode_check, ok := patroni_responce["synchronous_mode"].(bool) + if !ok { + return false, fmt.Errorf("failed to cast synchronous_mode field from patroni responce") } - log.S().Infof("Execute %v successfully: %v", cmd, string(buf)) + if mode_check { + return true, nil + } + + return false, nil - return nil } func (patroniAttack) Recover(exp core.Experiment, _ Environment) error { diff --git a/pkg/server/utils/http_requests.go b/pkg/server/utils/http_requests.go new file mode 100644 index 00000000..87ff03af --- /dev/null +++ b/pkg/server/utils/http_requests.go @@ -0,0 +1,135 @@ +// Copyright 2023 Chaos Mesh Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package utils + +import ( + "bytes" + "fmt" + "io" + "net/http" + + "github.com/pingcap/log" + "github.com/pkg/errors" + "github.com/tidwall/gjson" +) + +type PatroniInfo struct { + Master string + Replicas []string + SyncStandby []string + Status []string +} + +func GetPatroniInfo(address string) (PatroniInfo, error) { + res, err := http.Get(fmt.Sprintf("http://%v:8008/cluster", address)) + if err != nil { + err = errors.Errorf("failed to get patroni status: %v", err) + return PatroniInfo{}, errors.WithStack(err) + } + + defer res.Body.Close() + + buf, err := io.ReadAll(res.Body) + if err != nil { + err = errors.Errorf("failed to read responce: %v", err) + return PatroniInfo{}, errors.WithStack(err) + } + + data := string(buf) + + patroniInfo := PatroniInfo{} + + members := gjson.Get(data, "members") + + for _, member := range members.Array() { + switch member.Get("role").Str { + case "leader": + patroniInfo.Master = member.Get("name").Str + patroniInfo.Status = append(patroniInfo.Status, member.Get("state").Str) + case "replica": + patroniInfo.Replicas = append(patroniInfo.Replicas, member.Get("name").Str) + patroniInfo.Status = append(patroniInfo.Status, member.Get("state").Str) + case "sync_standby": + patroniInfo.SyncStandby = append(patroniInfo.SyncStandby, member.Get("name").Str) + patroniInfo.Status = append(patroniInfo.Status, member.Get("state").Str) + + } + } + + log.Info(fmt.Sprintf("patroni info: master %v, replicas %v, sync_standy %s, statuses %v\n", patroniInfo.Master, patroniInfo.Replicas, + patroniInfo.SyncStandby, patroniInfo.Status)) + + return patroniInfo, nil + +} + +func MakeHTTPRequest(method string, address string, port int64, path string, body []byte, user string, password string) ([]byte, error) { + + url := fmt.Sprintf("http://%v:%v/%v", address, port, path) + + var request *http.Request + + var resp *http.Response + + var err error + + switch method { + case http.MethodPost: + request, err = http.NewRequest("POST", url, bytes.NewBuffer(body)) + if err != nil { + err = errors.Errorf("failed to post request %v: %v", url, err) + return nil, errors.WithStack(err) + } + + case http.MethodGet: + request, err = http.NewRequest("GET", url, nil) + if err != nil { + err = errors.Errorf("failed to get request %v: %v", url, err) + return nil, errors.WithStack(err) + } + } + + if user != "" && password != "" { + request.Header.Set("Content-Type", "application/json") + request.SetBasicAuth(user, password) + } + + client := &http.Client{} + resp, err = client.Do(request) + if err != nil { + err = errors.Errorf("failed to exec %v request %v: %v", method, url, err) + return nil, errors.WithStack(err) + } + + defer resp.Body.Close() + + if resp.StatusCode != 200 && resp.StatusCode != 202 { + //to simplify diagnostics + buf, err := io.ReadAll(resp.Body) + if err != nil { + err = errors.Errorf("failed to read from %s responce: status code %v, responce %v, error %v", path, resp.StatusCode, resp.Body, err) + return nil, err + } + err = errors.Errorf("failed to exec %v request: status code %v, responce %v", path, resp.StatusCode, buf) + return nil, errors.WithStack(err) + } + + buf, err := io.ReadAll(resp.Body) + if err != nil { + err = errors.Errorf("failed to read %v from %s responce: %v", resp.Body, path, err) + return nil, errors.WithStack(err) + } + + return buf, nil +} diff --git a/pkg/server/utils/status.go b/pkg/server/utils/status.go deleted file mode 100644 index f49bd033..00000000 --- a/pkg/server/utils/status.go +++ /dev/null @@ -1,66 +0,0 @@ -// Copyright 2023 Chaos Mesh Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// See the License for the specific language governing permissions and -// limitations under the License. -package utils - -import ( - "fmt" - "io" - "net/http" - - "github.com/pingcap/log" - "github.com/pkg/errors" - "github.com/tidwall/gjson" -) - -type PatroniInfo struct { - Master string - Replicas []string - Status []string -} - -func GetPatroniInfo(address string) (PatroniInfo, error) { - res, err := http.Get(fmt.Sprintf("http://%v:8008/cluster", address)) - if err != nil { - err = errors.Errorf("failed to get patroni status: %v", err) - return PatroniInfo{}, errors.WithStack(err) - } - - defer res.Body.Close() - - buf, err := io.ReadAll(res.Body) - if err != nil { - err = errors.Errorf("failed to read responce: %v", err) - return PatroniInfo{}, errors.WithStack(err) - } - - data := string(buf) - - patroniInfo := PatroniInfo{} - - members := gjson.Get(data, "members") - - for _, member := range members.Array() { - if member.Get("role").Str == "leader" { - patroniInfo.Master = member.Get("name").Str - patroniInfo.Status = append(patroniInfo.Status, member.Get("state").Str) - } else if member.Get("role").Str == "replica" || member.Get("role").Str == "sync_standby" { - patroniInfo.Replicas = append(patroniInfo.Replicas, member.Get("name").Str) - patroniInfo.Status = append(patroniInfo.Status, member.Get("state").Str) - } - } - - log.Info(fmt.Sprintf("patroni info: master %v, replicas %v, statuses %v\n", patroniInfo.Master, patroniInfo.Replicas, patroniInfo.Status)) - - return patroniInfo, nil - -} diff --git a/pkg/server/utils/system_unix.go b/pkg/server/utils/system_unix.go new file mode 100644 index 00000000..abe6acae --- /dev/null +++ b/pkg/server/utils/system_unix.go @@ -0,0 +1,27 @@ +// Copyright 2023 Chaos Mesh Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package utils + +import ( + "os" +) + +func GetLocalHostname() (string, error) { + hostname, err := os.Hostname() + if err != nil { + return "", err + } + + return hostname, nil +} diff --git a/pkg/utils/command_test.go b/pkg/utils/command_test.go index ae9d3883..12716746 100644 --- a/pkg/utils/command_test.go +++ b/pkg/utils/command_test.go @@ -4,12 +4,13 @@ // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // -// http://www.apache.org/licenses/LICENSE-2.0 +// http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // See the License for the specific language governing permissions and // limitations under the License. + package utils import (