-
Notifications
You must be signed in to change notification settings - Fork 63
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Add attack to Patroni PostgreSQL cluster (#229)
Signed-off-by: Nikita Savchenko <nikisavchenko@ozon.ru>
- Loading branch information
Nikita Savchenko
committed
Feb 2, 2023
1 parent
0fa7a8e
commit 3461e37
Showing
7 changed files
with
376 additions
and
0 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,97 @@ | ||
| // Copyright 2020 Chaos Mesh Authors. | ||
| // | ||
| // Licensed under the Apache License, Version 2.0 (the "License"); | ||
| // you may not use this file except in compliance with the License. | ||
| // You may obtain a copy of the License at | ||
| // | ||
| // http://www.apache.org/licenses/LICENSE-2.0 | ||
| // | ||
| // Unless required by applicable law or agreed to in writing, software | ||
| // distributed under the License is distributed on an "AS IS" BASIS, | ||
| // See the License for the specific language governing permissions and | ||
| // limitations under the License. | ||
|
|
||
| package attack | ||
|
|
||
| import ( | ||
| "fmt" | ||
| "time" | ||
|
|
||
| "github.com/spf13/cobra" | ||
| "go.uber.org/fx" | ||
|
|
||
| "github.com/chaos-mesh/chaosd/cmd/server" | ||
| "github.com/chaos-mesh/chaosd/pkg/core" | ||
| "github.com/chaos-mesh/chaosd/pkg/server/chaosd" | ||
| "github.com/chaos-mesh/chaosd/pkg/utils" | ||
| ) | ||
|
|
||
| func NewPatroniAttackCommand(uid *string) *cobra.Command { | ||
| options := core.NewPatroniCommand() | ||
| dep := fx.Options( | ||
| server.Module, | ||
| fx.Provide(func() *core.PatroniCommand { | ||
| options.UID = *uid | ||
| return options | ||
| }), | ||
| ) | ||
|
|
||
| cmd := &cobra.Command{ | ||
| Use: "patroni <subcommand>", | ||
| Short: "Patroni attack related commands", | ||
| } | ||
|
|
||
| cmd.AddCommand( | ||
| NewPatroniSwitchoverCommand(dep, options), | ||
| NewPatroniFailoverCommand(dep, options), | ||
| ) | ||
|
|
||
| cmd.PersistentFlags().StringVarP(&options.User, "user", "u", "patroni", "patroni cluster user") | ||
| cmd.PersistentFlags().StringVar(&options.Password, "password", "p", "patroni cluster password") | ||
|
|
||
| return cmd | ||
| } | ||
|
|
||
| func NewPatroniSwitchoverCommand(dep fx.Option, options *core.PatroniCommand) *cobra.Command { | ||
| cmd := &cobra.Command{ | ||
| Use: "switchover", | ||
| Short: "exec switchover, default without another attack. Warning! Command is not recover!", | ||
| Run: func(*cobra.Command, []string) { | ||
| options.Action = core.SwitchoverAction | ||
| utils.FxNewAppWithoutLog(dep, fx.Invoke(PatroniAttackF)).Run() | ||
| }, | ||
| } | ||
| cmd.Flags().StringVarP(&options.Address, "address", "a", "", "patroni cluster address, any of available hosts") | ||
| cmd.Flags().StringVarP(&options.Candidate, "candidate", "c", "", "switchover candidate, default random unit for replicas") | ||
| cmd.Flags().StringVarP(&options.Scheduled_at, "scheduled_at", "d", fmt.Sprintln(time.Now().Add(time.Second*60).Format(time.RFC3339)), "scheduled switchover, default now()+1 minute") | ||
|
|
||
| return cmd | ||
| } | ||
|
|
||
| func NewPatroniFailoverCommand(dep fx.Option, options *core.PatroniCommand) *cobra.Command { | ||
| cmd := &cobra.Command{ | ||
| Use: "failover", | ||
| Short: "exec failover, default without another attack", | ||
| Run: func(*cobra.Command, []string) { | ||
| options.Action = core.FailoverAction | ||
| utils.FxNewAppWithoutLog(dep, fx.Invoke(PatroniAttackF)).Run() | ||
| }, | ||
| } | ||
|
|
||
| cmd.Flags().StringVarP(&options.Address, "address", "a", "", "patroni cluster address, any of available hosts") | ||
| cmd.Flags().StringVarP(&options.Candidate, "leader", "c", "", "failover new leader, default random unit for replicas") | ||
| return cmd | ||
| } | ||
|
|
||
| func PatroniAttackF(options *core.PatroniCommand, chaos *chaosd.Server) { | ||
| if err := options.Validate(); err != nil { | ||
| utils.ExitWithError(utils.ExitBadArgs, err) | ||
| } | ||
|
|
||
| uid, err := chaos.ExecuteAttack(chaosd.PatroniAttack, options, core.CommandMode) | ||
| if err != nil { | ||
| utils.ExitWithError(utils.ExitError, err) | ||
| } | ||
|
|
||
| utils.NormalExit(fmt.Sprintf("Attack %s successfully to patroni address %s, uid: %s", options.Action, options.Address, uid)) | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,72 @@ | ||
| // Copyright 2020 Chaos Mesh Authors. | ||
| // | ||
| // Licensed under the Apache License, Version 2.0 (the "License"); | ||
| // you may not use this file except in compliance with the License. | ||
| // You may obtain a copy of the License at | ||
| // | ||
| // http://www.apache.org/licenses/LICENSE-2.0 | ||
| // | ||
| // Unless required by applicable law or agreed to in writing, software | ||
| // distributed under the License is distributed on an "AS IS" BASIS, | ||
| // See the License for the specific language governing permissions and | ||
| // limitations under the License. | ||
|
|
||
| package core | ||
|
|
||
| import ( | ||
| "encoding/json" | ||
|
|
||
| "github.com/pingcap/errors" | ||
| ) | ||
|
|
||
| const ( | ||
| SwitchoverAction = "switchover" | ||
| FailoverAction = "failover" | ||
| ) | ||
|
|
||
| var _ AttackConfig = &PatroniCommand{} | ||
|
|
||
| type PatroniCommand struct { | ||
| CommonAttackConfig | ||
|
|
||
| Address string `json:"address,omitempty"` | ||
| Candidate string `json:"candidate,omitempty"` | ||
| Leader string `json:"leader,omitempty"` | ||
| User string `json:"user,omitempty"` | ||
| Password string `json:"password,omitempty"` | ||
| Scheduled_at string `json:"scheduled_at,omitempty"` | ||
| RecoverCmd string `json:"recoverCmd,omitempty"` | ||
| } | ||
|
|
||
| func (p *PatroniCommand) Validate() error { | ||
| if err := p.CommonAttackConfig.Validate(); err != nil { | ||
| return err | ||
| } | ||
| if len(p.Address) == 0 { | ||
| return errors.New("address not provided") | ||
| } | ||
|
|
||
| if len(p.User) == 0 { | ||
| return errors.New("patroni user not provided") | ||
| } | ||
|
|
||
| if len(p.Password) == 0 { | ||
| return errors.New("patroni password not provided") | ||
| } | ||
|
|
||
| return nil | ||
| } | ||
|
|
||
| func (p PatroniCommand) RecoverData() string { | ||
| data, _ := json.Marshal(p) | ||
|
|
||
| return string(data) | ||
| } | ||
|
|
||
| func NewPatroniCommand() *PatroniCommand { | ||
| return &PatroniCommand{ | ||
| CommonAttackConfig: CommonAttackConfig{ | ||
| Kind: PatroniAttack, | ||
| }, | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,131 @@ | ||
| // Copyright 2023 Chaos Mesh Authors. | ||
| // | ||
| // Licensed under the Apache License, Version 2.0 (the "License"); | ||
| // you may not use this file except in compliance with the License. | ||
| // You may obtain a copy of the License at | ||
| // | ||
| // http://www.apache.org/licenses/LICENSE-2.0 | ||
| // | ||
| // Unless required by applicable law or agreed to in writing, software | ||
| // distributed under the License is distributed on an "AS IS" BASIS, | ||
| // See the License for the specific language governing permissions and | ||
| // limitations under the License. | ||
|
|
||
| package chaosd | ||
|
|
||
| import ( | ||
| "bytes" | ||
| "encoding/json" | ||
| "fmt" | ||
| "io" | ||
| "math/rand" | ||
| "net/http" | ||
|
|
||
| "github.com/chaos-mesh/chaosd/pkg/core" | ||
| "github.com/chaos-mesh/chaosd/pkg/server/utils" | ||
| "github.com/pingcap/errors" | ||
| "github.com/pingcap/log" | ||
| ) | ||
|
|
||
| type patroniAttack struct{} | ||
|
|
||
| var PatroniAttack AttackType = patroniAttack{} | ||
|
|
||
| func (patroniAttack) Attack(options core.AttackConfig, _ Environment) error { | ||
| attack := options.(*core.PatroniCommand) | ||
|
|
||
| candidate := attack.Candidate | ||
|
|
||
| leader := attack.Leader | ||
|
|
||
| var scheduled_at string | ||
|
|
||
| var url string | ||
|
|
||
| values := make(map[string]string) | ||
|
|
||
| patroniInfo, err := utils.GetPatroniInfo(attack.Address) | ||
| if err != nil { | ||
| err = errors.Errorf("failed to get patroni info for %v: %v", options.String(), err) | ||
| return errors.WithStack(err) | ||
| } | ||
|
|
||
| if len(patroniInfo.Replicas) == 0 { | ||
| err = errors.Errorf("failed to get available replicas. Please, check your cluster") | ||
| return errors.WithStack(err) | ||
| } | ||
|
|
||
| if candidate == "" { | ||
| candidate = patroniInfo.Replicas[rand.Intn(len(patroniInfo.Replicas))] | ||
| } | ||
|
|
||
| if leader == "" { | ||
| leader = patroniInfo.Master | ||
| } | ||
|
|
||
| switch options.String() { | ||
| case "switchover": | ||
|
|
||
| scheduled_at = attack.Scheduled_at | ||
|
|
||
| values = map[string]string{"leader": leader, "scheduled_at": scheduled_at} | ||
|
|
||
| log.Info(fmt.Sprintf("Switchover will be done from %v to another available replica in %v", patroniInfo.Master, scheduled_at)) | ||
|
|
||
| case "failover": | ||
|
|
||
| values = map[string]string{"candidate": candidate} | ||
|
|
||
| log.Info(fmt.Sprintf("Failover will be done from %v to %v", patroniInfo.Master, candidate)) | ||
|
|
||
| } | ||
|
|
||
| patroniAddr := attack.Address | ||
|
|
||
| cmd := options.String() | ||
|
|
||
| data, err := json.Marshal(values) | ||
| if err != nil { | ||
| err = errors.Errorf("failed to marshal data: %v", values) | ||
| return errors.WithStack(err) | ||
| } | ||
|
|
||
| url = fmt.Sprintf("http://%v:8008/%v", patroniAddr, cmd) | ||
|
|
||
| request, err := http.NewRequest("POST", url, bytes.NewBuffer(data)) | ||
| if err != nil { | ||
| err = errors.Errorf("failed to %v: %v", cmd, err) | ||
| return errors.WithStack(err) | ||
| } | ||
|
|
||
| request.Header.Set("Content-Type", "application/json") | ||
| request.SetBasicAuth(attack.User, attack.Password) | ||
|
|
||
| client := &http.Client{} | ||
| resp, error := client.Do(request) | ||
| if error != nil { | ||
| err = errors.Errorf("failed to %v: %v", cmd, err) | ||
| return errors.WithStack(err) | ||
| } | ||
|
|
||
| defer resp.Body.Close() | ||
|
|
||
| buf, err := io.ReadAll(resp.Body) | ||
| if err != nil { | ||
| err = errors.Errorf("failed to read %v responce: %v", cmd, err) | ||
| return errors.WithStack(err) | ||
| } | ||
|
|
||
| if resp.StatusCode != 200 && resp.StatusCode != 202 { | ||
| err = errors.Errorf("failed to %v: status code %v, responce %v", cmd, resp.StatusCode, string(buf)) | ||
| return errors.WithStack(err) | ||
| } | ||
|
|
||
| log.S().Infof("Execute %v successfully: %v", cmd, string(buf)) | ||
|
|
||
| return nil | ||
| } | ||
|
|
||
| func (patroniAttack) Recover(exp core.Experiment, _ Environment) error { | ||
| return nil | ||
| } |
Oops, something went wrong.