etcdserver: Add downgrade support POC

etcdserver/api/capability.go

@@ -17,6 +17,7 @@ package api
 import (
 	"sync"
 
+	"go.etcd.io/etcd/etcdserver/api/membership"
 	"go.etcd.io/etcd/version"
 	"go.uber.org/zap"
 
@@ -65,7 +66,7 @@ func UpdateCapability(lg *zap.Logger, v *semver.Version) {
 		return
 	}
 	enableMapMu.Lock()
-	if curVersion != nil && !curVersion.LessThan(*v) {
+	if curVersion != nil && !membership.IsVersionChangable(v, curVersion) {
 		enableMapMu.Unlock()
 		return
 	}

etcdserver/api/membership/cluster.go

@@ -816,18 +816,21 @@ func mustDetectDowngrade(lg *zap.Logger, cv *semver.Version) {
 	// only keep major.minor version for comparison against cluster version
 	lv = &semver.Version{Major: lv.Major, Minor: lv.Minor}
 	if cv != nil && lv.LessThan(*cv) {
-		if lg != nil {
-			lg.Fatal(
-				"invalid downgrade; server version is lower than determined cluster version",
-				zap.String("current-server-version", version.Version),
-				zap.String("determined-cluster-version", version.Cluster(cv.String())),
-			)
+		if IsVersionChangable(cv, lv) {
+			plog.Infof("cluster is downgrading to current version: %s from determined cluster version: %s).", version.Version, version.Cluster(cv.String()))
 		} else {
-			plog.Fatalf("cluster cannot be downgraded (current version: %s is lower than determined cluster version: %s).", version.Version, version.Cluster(cv.String()))
+			plog.Fatalf("cluster cannot be downgraded (current version: %s is too much lower than determined cluster version: %s).", version.Version, version.Cluster(cv.String()))
 		}
 	}
 }
 
+func IsVersionChangable(cv *semver.Version, lv *semver.Version) bool {
+	if (cv.Major == lv.Major) && ((cv.Minor-lv.Minor) == 1 || lv.Minor > cv.Minor) {
+		return true
+	}
+	return false
+}
+
 // IsLocalMemberLearner returns if the local member is raft learner
 func (c *RaftCluster) IsLocalMemberLearner() bool {
 	c.Lock()

etcdserver/cluster_util.go

@@ -224,19 +224,40 @@ func decideClusterVersion(lg *zap.Logger, vers map[string]*version.Versions) *se
 	return cv
 }
 
+// canUpdateClusterVersion verify whether to update cluster version:
+// - if --experimental-enable-cluster-downgrade is set to false (default),
+//   update cluster version only if the decided version is greater than
+//   the current cluster version
+// - if --experimental-enable-cluster-downgrade is set to true,
+//    update cluster version if the decided version is +1 or -1 minor
+//    version difference than current cluster version
+func canUpdateClusterVersion(clusterDowngradeEnabled bool, decidedClusterVersion *semver.Version, currentClusterVersion *semver.Version) bool {
+	if (!clusterDowngradeEnabled && currentClusterVersion.LessThan(*decidedClusterVersion)) ||
+		(clusterDowngradeEnabled && membership.IsVersionChangable(currentClusterVersion, decidedClusterVersion)) {
+		return true
+	}
+	return false
+}
+
 // isCompatibleWithCluster return true if the local member has a compatible version with
 // the current running cluster.
 // The version is considered as compatible when at least one of the other members in the cluster has a
 // cluster version in the range of [MinClusterVersion, Version] and no known members has a cluster version
 // out of the range.
 // We set this rule since when the local member joins, another member might be offline.
-func isCompatibleWithCluster(lg *zap.Logger, cl *membership.RaftCluster, local types.ID, rt http.RoundTripper) bool {
+// When cluster downgrade support is enabled, set maximum cluster version to be 1 minor version higher to
+// to allow current local member to join a cluster at 1 minor version high.
+func isCompatibleWithCluster(lg *zap.Logger, cl *membership.RaftCluster, local types.ID, rt http.RoundTripper, clusterDowngradeEnabled bool) bool {
 	vers := getVersions(lg, cl, local, rt)
 	minV := semver.Must(semver.NewVersion(version.MinClusterVersion))
 	maxV := semver.Must(semver.NewVersion(version.Version))
+	allowedClusterMinor := maxV.Minor
+	if clusterDowngradeEnabled {
+		allowedClusterMinor = maxV.Minor + 1
+	}
 	maxV = &semver.Version{
 		Major: maxV.Major,
-		Minor: maxV.Minor,
+		Minor: allowedClusterMinor,
 	}
 	return isCompatibleWithVers(lg, vers, local, minV, maxV)
 }

etcdserver/config.go

@@ -157,6 +157,8 @@ type ServerConfig struct {
 	LeaseCheckpointInterval time.Duration
 
 	EnableGRPCGateway bool
+
+	EnableClusterDowngrade bool
 }
 
 // VerifyBootstrap sanity-checks the initial config for bootstrap case

etcdserver/server.go

@@ -364,7 +364,7 @@ func NewServer(cfg ServerConfig) (srv *EtcdServer, err error) {
 		if err = membership.ValidateClusterAndAssignIDs(cfg.Logger, cl, existingCluster); err != nil {
 			return nil, fmt.Errorf("error validating peerURLs %s: %v", existingCluster, err)
 		}
-		if !isCompatibleWithCluster(cfg.Logger, cl, cl.MemberByName(cfg.Name).ID, prt) {
+		if !isCompatibleWithCluster(cfg.Logger, cl, cl.MemberByName(cfg.Name).ID, prt, cfg.EnableClusterDowngrade) {
 			return nil, fmt.Errorf("incompatible with current running cluster")
 		}
 
@@ -2555,9 +2555,10 @@ func (s *EtcdServer) monitorVersions() {
 			continue
 		}
 
-		// update cluster version only if the decided version is greater than
-		// the current cluster version
-		if v != nil && s.cluster.Version().LessThan(*v) {
+		// Original etcd v3.1.26 only update cluster version if the decided version is
+		// greater than the current cluster version, in this patched etcd, we relax the rule
+		// and allow +1 or -1 minor version cluster version change
+		if v != nil && canUpdateClusterVersion(s.Cfg.EnableClusterDowngrade, v, s.cluster.Version()) {
 			s.goAttach(func() { s.updateClusterVersion(v.String()) })
 		}
 	}