From 67e86c0dc828d368e36dbb42ef8f2b433b7960d6 Mon Sep 17 00:00:00 2001 From: Nikita Savchenko Date: Tue, 24 Jan 2023 12:51:05 +0400 Subject: [PATCH] Add attack to Patroni PostgreSQL cluster (#229) Signed-off-by: Nikita Savchenko --- cmd/attack/patroni.go | 97 ++++++++++++++++++++++++++ go.mod | 3 + go.sum | 6 ++ pkg/core/experiment.go | 1 + pkg/core/patroni.go | 66 ++++++++++++++++++ pkg/server/chaosd/patroni.go | 128 +++++++++++++++++++++++++++++++++++ pkg/server/utils/status.go | 54 +++++++++++++++ 7 files changed, 355 insertions(+) create mode 100644 cmd/attack/patroni.go create mode 100644 pkg/core/patroni.go create mode 100644 pkg/server/chaosd/patroni.go create mode 100644 pkg/server/utils/status.go diff --git a/cmd/attack/patroni.go b/cmd/attack/patroni.go new file mode 100644 index 00000000..84260994 --- /dev/null +++ b/cmd/attack/patroni.go @@ -0,0 +1,97 @@ +// Copyright 2020 Chaos Mesh Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package attack + +import ( + "fmt" + "time" + + "github.com/spf13/cobra" + "go.uber.org/fx" + + "github.com/chaos-mesh/chaosd/cmd/server" + "github.com/chaos-mesh/chaosd/pkg/core" + "github.com/chaos-mesh/chaosd/pkg/server/chaosd" + "github.com/chaos-mesh/chaosd/pkg/utils" +) + +func NewPatroniAttackCommand(uid *string) *cobra.Command { + options := core.NewPatroniCommand() + dep := fx.Options( + server.Module, + fx.Provide(func() *core.PatroniCommand { + options.UID = *uid + return options + }), + ) + + cmd := &cobra.Command{ + Use: "patroni ", + Short: "Patroni attack related commands", + } + + cmd.AddCommand( + NewPatroniSwitchoverCommand(dep, options), + NewPatroniFailoverCommand(dep, options), + ) + + cmd.PersistentFlags().StringVarP(&options.User, "user", "u", "patroni", "patroni cluster user") + cmd.PersistentFlags().StringVar(&options.Password, "password", "p", "patroni cluster password") + + return cmd +} + +func NewPatroniSwitchoverCommand(dep fx.Option, options *core.PatroniCommand) *cobra.Command { + cmd := &cobra.Command{ + Use: "switchover", + Short: "exec switchover, default without another attack. Warning! Command is not recover!", + Run: func(*cobra.Command, []string) { + options.Action = core.SwitchoverAction + utils.FxNewAppWithoutLog(dep, fx.Invoke(PatroniAttackF)).Run() + }, + } + cmd.Flags().StringVarP(&options.Address, "address", "a", "", "patroni cluster address, any of available hosts") + cmd.Flags().StringVarP(&options.Candidate, "candidate", "c", "", "switchover candidate, default random unit for replicas") + cmd.Flags().StringVarP(&options.Scheduled_at, "scheduled_at", "d", fmt.Sprintln(time.Now().Add(time.Second*60).Format(time.RFC3339)), "scheduled switchover, default now()+1 minute") + + return cmd +} + +func NewPatroniFailoverCommand(dep fx.Option, options *core.PatroniCommand) *cobra.Command { + cmd := &cobra.Command{ + Use: "failover", + Short: "exec failover, default without another attack", + Run: func(*cobra.Command, []string) { + options.Action = core.FailoverAction + utils.FxNewAppWithoutLog(dep, fx.Invoke(PatroniAttackF)).Run() + }, + } + + cmd.Flags().StringVarP(&options.Address, "address", "a", "", "patroni cluster address, any of available hosts") + cmd.Flags().StringVarP(&options.Candidate, "leader", "c", "", "failover new leader, default random unit for replicas") + return cmd +} + +func PatroniAttackF(options *core.PatroniCommand, chaos *chaosd.Server) { + if err := options.Validate(); err != nil { + utils.ExitWithError(utils.ExitBadArgs, err) + } + + uid, err := chaos.ExecuteAttack(chaosd.PatroniAttack, options, core.CommandMode) + if err != nil { + utils.ExitWithError(utils.ExitError, err) + } + + utils.NormalExit(fmt.Sprintf("Attack %s successfully to patroni address %s, uid: %s", options.Action, options.Address, uid)) +} diff --git a/go.mod b/go.mod index db9a9c28..a01397de 100644 --- a/go.mod +++ b/go.mod @@ -32,6 +32,7 @@ require ( github.com/swaggo/files v0.0.0-20220610200504-28940afbdbfe github.com/swaggo/gin-swagger v1.5.0 github.com/swaggo/swag v1.8.3 + github.com/tidwall/gjson v1.14.4 go.uber.org/fx v1.17.1 go.uber.org/zap v1.21.0 google.golang.org/grpc v1.40.0 @@ -122,6 +123,8 @@ require ( github.com/romana/ipset v1.0.0 // indirect github.com/romana/rlog v0.0.0-20171115192701-f018bc92e7d7 // indirect github.com/sirupsen/logrus v1.8.1 // indirect + github.com/tidwall/match v1.1.1 // indirect + github.com/tidwall/pretty v1.2.0 // indirect github.com/tklauser/go-sysconf v0.3.10 // indirect github.com/tklauser/numcpus v0.4.0 // indirect github.com/ugorji/go/codec v1.2.7 // indirect diff --git a/go.sum b/go.sum index c6271417..f8e5caf9 100644 --- a/go.sum +++ b/go.sum @@ -1029,6 +1029,12 @@ github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635/go.mod h1:hkRG github.com/syndtr/goleveldb v1.0.1-0.20200815110645-5c35d600f0ca/go.mod h1:u2MKkTVTVJWe5D1rCvame8WqhBd88EuIwODJZ1VHCPM= github.com/syndtr/goleveldb v1.0.1-0.20210305035536-64b5b1c73954/go.mod h1:u2MKkTVTVJWe5D1rCvame8WqhBd88EuIwODJZ1VHCPM= github.com/tchap/go-patricia v2.2.6+incompatible/go.mod h1:bmLyhP68RS6kStMGxByiQ23RP/odRBOTVjwp2cDyi6I= +github.com/tidwall/gjson v1.14.4 h1:uo0p8EbA09J7RQaflQ1aBRffTR7xedD2bcIVSYxLnkM= +github.com/tidwall/gjson v1.14.4/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= +github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA= +github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= +github.com/tidwall/pretty v1.2.0 h1:RWIZEg2iJ8/g6fDDYzMpobmaoGh5OLl4AXtGUGPcqCs= +github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= github.com/tinylib/msgp v1.0.2/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDWklFE= github.com/tklauser/go-sysconf v0.3.10 h1:IJ1AZGZRWbY8T5Vfk04D9WOA5WSejdflXxP03OUqALw= github.com/tklauser/go-sysconf v0.3.10/go.mod h1:C8XykCvCb+Gn0oNCWPIlcb0RuglQTYaQ2hGm7jmxEFk= diff --git a/pkg/core/experiment.go b/pkg/core/experiment.go index c9008a8a..caa1c498 100644 --- a/pkg/core/experiment.go +++ b/pkg/core/experiment.go @@ -43,6 +43,7 @@ const ( FileAttack = "file" HTTPAttack = "http" VMAttack = "vm" + PatroniAttack = "patroni" UserDefinedAttack = "userDefined" ) diff --git a/pkg/core/patroni.go b/pkg/core/patroni.go new file mode 100644 index 00000000..222f5302 --- /dev/null +++ b/pkg/core/patroni.go @@ -0,0 +1,66 @@ +// Copyright 2020 Chaos Mesh Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package core + +import ( + "encoding/json" + + "github.com/pingcap/errors" +) + +const ( + SwitchoverAction = "switchover" + FailoverAction = "failover" +) + +var _ AttackConfig = &PatroniCommand{} + +type PatroniCommand struct { + CommonAttackConfig + + Address string `json:"address,omitempty"` + Candidate string `json:"candidate,omitempty"` + Leader string `json:"leader,omitempty"` + User string `json:"user,omitempty"` + Password string `json:"password,omitempty"` + Scheduled_at string `json:"scheduled_at,omitempty"` + RecoverCmd string `json:"recoverCmd,omitempty"` +} + +func (p *PatroniCommand) Validate() error { + if err := p.CommonAttackConfig.Validate(); err != nil { + return err + } + if len(p.Address) == 0 { + return errors.New("address not provided") + } + + // TODO: validate signal + + return nil +} + +func (p PatroniCommand) RecoverData() string { + data, _ := json.Marshal(p) + + return string(data) +} + +func NewPatroniCommand() *PatroniCommand { + return &PatroniCommand{ + CommonAttackConfig: CommonAttackConfig{ + Kind: PatroniAttack, + }, + } +} diff --git a/pkg/server/chaosd/patroni.go b/pkg/server/chaosd/patroni.go new file mode 100644 index 00000000..3d18db53 --- /dev/null +++ b/pkg/server/chaosd/patroni.go @@ -0,0 +1,128 @@ +package chaosd + +import ( + "bytes" + "encoding/json" + "fmt" + "io" + "math/rand" + "net/http" + + "github.com/chaos-mesh/chaosd/pkg/core" + "github.com/chaos-mesh/chaosd/pkg/server/utils" + "github.com/pingcap/errors" + "github.com/pingcap/log" +) + +type patroniAttack struct{} + +var PatroniAttack AttackType = patroniAttack{} + +func (patroniAttack) Attack(options core.AttackConfig, _ Environment) error { + attack := options.(*core.PatroniCommand) + + candidate := attack.Candidate + + leader := attack.Leader + + var scheduled_at string + + var url string + + var availableReplicas []string + + values := make(map[string]string) + + patroniInfo, err := utils.GetPatroniInfo(attack.Address) + if err != nil { + err = errors.Errorf("failed to get patroni info for : %v", options.String(), err) + return errors.WithStack(err) + } + + for _, replica := range patroniInfo.Replicas { + if replica != attack.Address { + availableReplicas = append(availableReplicas, replica) + } + } + + if len(availableReplicas) == 0 { + err = errors.Errorf("failed to get available replics. Please, choose another host") + return errors.WithStack(err) + } + + if candidate == "" { + + candidate = availableReplicas[rand.Intn(len(availableReplicas))] + + } + + if leader == "" { + leader = patroniInfo.Master + } + + switch options.String() { + case "switchover": + + scheduled_at = attack.Scheduled_at + + values = map[string]string{"leader": leader, "scheduled_at": scheduled_at} + + log.Info(fmt.Sprintf("Switchover will be done from %v to another available replica in %v", patroniInfo.Master, scheduled_at)) + + case "failover": + + values = map[string]string{"candidate": candidate} + + log.Info(fmt.Sprintf("Failover will be done from %v to %v", patroniInfo.Master, candidate)) + + } + + patroniAddr := attack.Address + + cmd := options.String() + + data, err := json.Marshal(values) + if err != nil { + err = errors.Errorf("failed to marshal data: %v", values) + return errors.WithStack(err) + } + + url = fmt.Sprintf("http://%v:8008/%v", patroniAddr, cmd) + + request, err := http.NewRequest("POST", url, bytes.NewBuffer(data)) + if err != nil { + err = errors.Errorf("failed to %v: %v", cmd, err) + return errors.WithStack(err) + } + + request.Header.Set("Content-Type", "application/json") + request.SetBasicAuth(attack.User, attack.Password) + + client := &http.Client{} + resp, error := client.Do(request) + if error != nil { + err = errors.Errorf("failed to %v: %v", cmd, err) + return errors.WithStack(err) + } + + defer resp.Body.Close() + + buf, err := io.ReadAll(resp.Body) + if err != nil { + err = errors.Errorf("failed to read %v responce: %v", cmd, err) + return errors.WithStack(err) + } + + if resp.StatusCode != 200 && resp.StatusCode != 202 { + err = errors.Errorf("failed to %v: status code %v, responce %v", cmd, resp.StatusCode, string(buf)) + return errors.WithStack(err) + } + + log.S().Infof("Execute %v successfully: %v", cmd, string(buf)) + + return nil +} + +func (patroniAttack) Recover(exp core.Experiment, _ Environment) error { + return nil +} diff --git a/pkg/server/utils/status.go b/pkg/server/utils/status.go new file mode 100644 index 00000000..a570d441 --- /dev/null +++ b/pkg/server/utils/status.go @@ -0,0 +1,54 @@ +package utils + +import ( + "fmt" + "io" + "net/http" + + "github.com/pingcap/log" + "github.com/pkg/errors" + "github.com/tidwall/gjson" +) + +type PatroniInfo struct { + Master string + Replicas []string + Status []string +} + +func GetPatroniInfo(address string) (PatroniInfo, error) { + res, err := http.Get(fmt.Sprintf("http://%v:8008/cluster", address)) + if err != nil { + err = errors.Errorf("failed to get patroni status: %v", err) + return PatroniInfo{}, errors.WithStack(err) + } + + defer res.Body.Close() + + buf, err := io.ReadAll(res.Body) + if err != nil { + err = errors.Errorf("failed to read responce: %v", err) + return PatroniInfo{}, errors.WithStack(err) + } + + data := string(buf) + + patroniInfo := PatroniInfo{} + + members := gjson.Get(data, "members") + + for _, member := range members.Array() { + if member.Get("role").Str == "leader" { + patroniInfo.Master = member.Get("name").Str + patroniInfo.Status = append(patroniInfo.Status, member.Get("state").Str) + } else if member.Get("role").Str == "replica" || member.Get("role").Str == "sync_standby" { + patroniInfo.Replicas = append(patroniInfo.Replicas, member.Get("name").Str) + patroniInfo.Status = append(patroniInfo.Status, member.Get("state").Str) + } + } + + log.Info(fmt.Sprintf("patroni info: master %v, replicas %v, statuses %v\n", patroniInfo.Master, patroniInfo.Replicas, patroniInfo.Status)) + + return patroniInfo, nil + +}