Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Etcd Cluster Downgrade #11362

Closed
wants to merge 25 commits into from
Closed
Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
25 commits
Select commit Hold shift + click to select a range
9298f84
etcdserver: Add downgrade support POC
wenjiaswe Oct 25, 2019
62280cc
etcdserver: add downgrade proto api
YoyinZyc Oct 24, 2019
1aa02da
clientV3: add downgrade api in maintenance
YoyinZyc Oct 24, 2019
336cef4
v3rpc: add downgrader api in maintenance server
YoyinZyc Oct 24, 2019
9da21e9
membership: add downgrade field into raftCluster
YoyinZyc Oct 24, 2019
2023979
etcdserver: implement EtcdServer.Downgrade; implement downgrade valid…
YoyinZyc Oct 24, 2019
5182d5e
membership: add downgrade backend support
YoyinZyc Oct 24, 2019
a81969f
membership: recover downgrade from backend when restart the server or…
YoyinZyc Oct 26, 2019
5df9185
etcdserver: add downgrade start and downgrade cancel
YoyinZyc Oct 28, 2019
1d80e74
etcdserver: add monitorDowngrade for monitoring the downgrade status
YoyinZyc Oct 28, 2019
4e429b7
etcdctl: add downgrade commands.
YoyinZyc Oct 29, 2019
fa150ce
etcdserver: update downgrade check for version when starting/restarti…
YoyinZyc Oct 31, 2019
ce75238
etcdserver: add unit&integration test for downgrade; etcdctl: add pri…
YoyinZyc Nov 6, 2019
87aaab6
etcdserver: add new http handler to fetch the downgrade status of a c…
YoyinZyc Nov 11, 2019
dbc383a
e2e: add etcd downgrade/upgrade test
YoyinZyc Nov 11, 2019
f01694a
test: ctl_downgrade_test
wenjiaswe Nov 11, 2019
2311bc4
e2e: fix ctl_v3_downgrade_test; etcdctl: update output messages
YoyinZyc Nov 12, 2019
fe53e35
etcdserver: remove Downgrade Action Status in rpc.proto
YoyinZyc Nov 12, 2019
5c7f7ee
etcdserver: change downgrade raft request type from ConfChange to Int…
YoyinZyc Nov 13, 2019
61082c8
etcdserver: allow previous binary to restart when downgrade. add new …
YoyinZyc Nov 14, 2019
197a8a6
etcdserver: fix TestCtlV3Migrate nil pointer issue
YoyinZyc Nov 14, 2019
6efeb79
etcdserver: change "/downgrade/enabled" endpoint to serve linearized …
YoyinZyc Nov 16, 2019
f961ac9
etcdserver: separate function isValidDowngrade from mustDetectDowngra…
YoyinZyc Nov 19, 2019
00a36b4
etcdserver: define error string and link grpc error code.
YoyinZyc Nov 23, 2019
22ddb23
etcdserver: add new raft internal request DowngradeInfoSetRequest
YoyinZyc Dec 16, 2019
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
e2e: fix ctl_v3_downgrade_test; etcdctl: update output messages
  • Loading branch information
YoyinZyc committed Dec 12, 2019
commit 2311bc4c58cbe48f628843c05dc26a2c8a31ceac
10 changes: 5 additions & 5 deletions etcdctl/ctlv3/command/downgrade_command.go
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@ func NewDowngradeCommand() *cobra.Command {
}

dc.AddCommand(NewDowngradeValidateCommand())
dc.AddCommand(NewDowngradeStartCommand())
dc.AddCommand(NewDowngradeEnableCommand())
dc.AddCommand(NewDowngradeCancelCommand())

return dc
Expand All @@ -44,8 +44,8 @@ func NewDowngradeValidateCommand() *cobra.Command {
return cc
}

// NewDowngradeStartCommand returns the cobra command for "downgrade start"
func NewDowngradeStartCommand() *cobra.Command {
// NewDowngradeEnableCommand returns the cobra command for "downgrade enable"
func NewDowngradeEnableCommand() *cobra.Command {
cc := &cobra.Command{
Use: "enable <targetVersion>",
Short: "Enable the cluster to downgrade to target version",
Expand All @@ -67,7 +67,7 @@ func NewDowngradeCancelCommand() *cobra.Command {
// downgradeValidateCommandFunc executes the "downgrade validate" command
func downgradeValidateCommandFunc(cmd *cobra.Command, args []string) {
if len(args) < 1 {
ExitWithError(ExitBadArgs, errors.New("target version not provided"))
ExitWithError(ExitBadArgs, errors.New("no target version input"))
}

if len(args) > 1 {
Expand All @@ -94,7 +94,7 @@ func downgradeValidateCommandFunc(cmd *cobra.Command, args []string) {
// downgradeEnableCommandFunc executes the "downgrade start" command
func downgradeEnableCommandFunc(cmd *cobra.Command, args []string) {
if len(args) < 1 {
ExitWithError(ExitBadArgs, errors.New("target version not provided"))
ExitWithError(ExitBadArgs, errors.New("no target version input"))
}

if len(args) > 1 {
Expand Down
4 changes: 2 additions & 2 deletions etcdctl/ctlv3/command/printer_fields.go
Original file line number Diff line number Diff line change
Expand Up @@ -229,7 +229,7 @@ func (p *fieldsPrinter) UserDelete(user string, r v3.AuthUserDeleteResponse) { p

func (p *fieldsPrinter) DowngradeValidate(r v3.DowngradeResponse) {
p.hdr(r.Header)
fmt.Println("The cluster can be downgrade to target version.")
fmt.Println("Validate succeeded")
fmt.Println(`"Current cluster version" :`, r.Version)
}

Expand All @@ -241,6 +241,6 @@ func (p *fieldsPrinter) DowngradeEnable(r v3.DowngradeResponse) {

func (p *fieldsPrinter) DowngradeCancel(r v3.DowngradeResponse) {
p.hdr(r.Header)
fmt.Println("Current downgrade job has been cancelled.")
fmt.Println("Cancelled")
fmt.Println(`"Current cluster version" :`, r.Version)
}
5 changes: 2 additions & 3 deletions etcdctl/ctlv3/command/printer_simple.go
Original file line number Diff line number Diff line change
Expand Up @@ -287,14 +287,13 @@ func (s *simplePrinter) UserList(r v3.AuthUserListResponse) {
}

func (s *simplePrinter) DowngradeValidate(r v3.DowngradeResponse) {
fmt.Printf("The cluster can be downgrade to target version. "+
"Current cluster version is %s\n", r.Version)
fmt.Printf("Validate succeeded. Current cluster version is %s\n", r.Version)
}

func (s *simplePrinter) DowngradeEnable(r v3.DowngradeResponse) {
fmt.Printf("The cluster is available to downgrade. Please wait until all servers are ready.\n")
}

func (s *simplePrinter) DowngradeCancel(r v3.DowngradeResponse) {
fmt.Printf("Current downgrade job has been cancelled.\n")
fmt.Printf("Cancelled.\n")
}
2 changes: 0 additions & 2 deletions etcdserver/api/membership/cluster.go
Original file line number Diff line number Diff line change
Expand Up @@ -271,10 +271,8 @@ func (c *RaftCluster) Recover(onSet func(*zap.Logger, *semver.Version)) {
if c.downgrade == nil {
d = &Downgrade{Enabled: false}
} else {
//fmt.Println("downgrade enabled:", d.Enabled)
d = &Downgrade{Enabled: c.downgrade.Enabled, TargetVersion: c.downgrade.TargetVersion}
}
fmt.Println("downgrade enabled:", d.Enabled)
mustDetectDowngrade(c.lg, c.version, d)
onSet(c.lg, c.version)

Expand Down
3 changes: 0 additions & 3 deletions etcdserver/api/membership/cluster_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -886,9 +886,6 @@ func TestMustDetectDowngrade(t *testing.T) {
lv = &semver.Version{Major: lv.Major, Minor: lv.Minor}
oneMinorHigher := &semver.Version{Major: lv.Major, Minor: lv.Minor + 1}
oneMinorLower := &semver.Version{Major: lv.Major, Minor: lv.Minor - 1}

fmt.Println(oneMinorHigher.String())
fmt.Println(oneMinorLower.String())
downgradeEnabledHigherVersion := &Downgrade{Enabled: true, TargetVersion: oneMinorHigher}
downgradeEnabledEqualVersion := &Downgrade{Enabled: true, TargetVersion: lv}
downgradeEnabledLowerVersion := &Downgrade{Enabled: true, TargetVersion: oneMinorLower}
Expand Down
15 changes: 8 additions & 7 deletions etcdserver/cluster_util.go
Original file line number Diff line number Diff line change
Expand Up @@ -177,9 +177,9 @@ func getVersions(lg *zap.Logger, cl *membership.RaftCluster, local types.ID, rt
return vers
}

// isDowngradeEnabled returns the downgrade enabled status in the given cluster.
// If the local server or any other remote server has already enable downgrade, return true.
// Otherwise, return false.
// isDowngradeEnabled returns the downgrade status in the given cluster.
// If the local server or any other remote server has already enable downgrade, return true and target version.
// Otherwise, return false and nil.
func isDowngradeEnabled(lg *zap.Logger, cl *membership.RaftCluster, local types.ID, rt http.RoundTripper) (bool, *semver.Version) {
members := cl.Members()
for _, m := range members {
Expand All @@ -202,8 +202,8 @@ func isDowngradeEnabled(lg *zap.Logger, cl *membership.RaftCluster, local types.
return false, nil
}

// getDowngradeStatus returns the downgrade enabled status of the given member
// via its peerURLs. Returns the last error if it fails to get the version.
// getDowngradeStatus returns the downgrade status of the given member
// via its peerURLs. Returns the last error if it fails to get it.
func getDowngradeStatus(lg *zap.Logger, m *membership.Member, rt http.RoundTripper) (*membership.Downgrade, error) {
cc := &http.Client{
Transport: rt,
Expand Down Expand Up @@ -343,8 +343,9 @@ func decideDowngradeStatus(lg *zap.Logger, targetVersion *semver.Version, vers m
// cluster version in the range of [MinClusterVersion, Version] and no known members has a cluster version
// out of the range.
// We set this rule since when the local member joins, another member might be offline.
// When cluster downgrade support is enabled, set maximum cluster version to be 1 minor version higher to
// to allow current local member to join a cluster at 1 minor version high.
// When cluster downgrade support is enabled, check whether the local version matches downgrade target version.
// If yes, set maximum cluster version to be 1 minor version higher to
// allow current local member to join a cluster at 1 minor version high.
func isCompatibleWithCluster(lg *zap.Logger, cl *membership.RaftCluster, local types.ID, rt http.RoundTripper) bool {
vers := getVersions(lg, cl, local, rt)
minV := semver.Must(semver.NewVersion(version.MinClusterVersion))
Expand Down
2 changes: 1 addition & 1 deletion etcdserver/server.go
Original file line number Diff line number Diff line change
Expand Up @@ -2777,7 +2777,7 @@ func (s *EtcdServer) raftStatus() raft.Status {
}

func (s *EtcdServer) downgradeEnable(ctx context.Context, v string) (*pb.DowngradeResponse, error) {
// validate downgrade capability before starting job
// validate downgrade capability before starting downgrade
if resp, err := s.downgradeValidate(ctx, v); err != nil {
return resp, err
}
Expand Down
4 changes: 2 additions & 2 deletions etcdserver/v3_server.go
Original file line number Diff line number Diff line change
Expand Up @@ -832,7 +832,7 @@ func (s *EtcdServer) downgradeValidate(ctx context.Context, v string) (*pb.Downg

resp.Version = cv.String()
if cv.LessThan(*targetVersion) {
err = errors.New("target version is higher than current cluster version")
err = errors.New("target version too high")
return nil, err
}

Expand All @@ -842,7 +842,7 @@ func (s *EtcdServer) downgradeValidate(ctx context.Context, v string) (*pb.Downg
}
if !membership.IsVersionChangable(cv, targetVersion) {
err = fmt.Errorf(
"target version violates the downgrade policy. "+
"target version too small. "+
"the cluster can only be downgraded to %s",
semver.Version{Major: cv.Major, Minor: cv.Minor - 1}.String())
return nil, err
Expand Down
4 changes: 2 additions & 2 deletions integration/v3_downgrade_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -67,7 +67,7 @@ func TestDowngradeValidate(t *testing.T) {
"validate downgrade against one minor version higher",
oneMinorHigher.String(),
false,
"target version is higher than current cluster version",
"target version too high",
},
{
"validate downgrade against one minor version lower",
Expand All @@ -79,7 +79,7 @@ func TestDowngradeValidate(t *testing.T) {
"validate downgrade against two minor version lower",
twoMinorLower.String(),
false,
"target version violates the downgrade policy",
"target version too small",
},
{
"validate downgrade against one minor version lower with patch version 1",
Expand Down
Loading