Etcd Cluster Downgrade

Documentation/dev-guide/api_reference_v3.md

@@ -74,6 +74,7 @@ This is a generated documentation. Please read the proto files for more.
 | HashKV | HashKVRequest | HashKVResponse | HashKV computes the hash of all MVCC keys up to a given revision. It only iterates "key" bucket in backend storage. |
 | Snapshot | SnapshotRequest | SnapshotResponse | Snapshot sends a snapshot of the entire backend from a member over a stream to a client. |
 | MoveLeader | MoveLeaderRequest | MoveLeaderResponse | MoveLeader requests current leader node to transfer its leadership to transferee. |
+| Downgrade | DowngradeRequest | DowngradeResponse | Downgrade requests downgrade, cancel downgrade on the cluster version. |
 
 
 
@@ -445,6 +446,24 @@ Empty field.
 
 
 
+##### message `DowngradeRequest` (etcdserver/etcdserverpb/rpc.proto)
+
+| Field | Description | Type |
+| ----- | ----------- | ---- |
+| action | action is the kind of downgrade request to issue. The action may VALIDATE the target version, DOWNGRADE the cluster version, or CANCEL the current downgrading job. | DowngradeAction |
+| version | version is the target version to downgrade. | string |
+
+
+
+##### message `DowngradeResponse` (etcdserver/etcdserverpb/rpc.proto)
+
+| Field | Description | Type |
+| ----- | ----------- | ---- |
+| header |  | ResponseHeader |
+| version | version is the current cluster version. | string |
+
+
+
 ##### message `HashKVRequest` (etcdserver/etcdserverpb/rpc.proto)
 
 | Field | Description | Type |

Documentation/dev-guide/apispec/swagger/rpc.swagger.json

@@ -988,6 +988,33 @@
         }
       }
     },
+    "/v3/maintenance/downgrade": {
+      "post": {
+        "tags": [
+          "Maintenance"
+        ],
+        "summary": "Downgrade requests downgrade, cancel downgrade on the cluster version.",
+        "operationId": "Downgrade",
+        "parameters": [
+          {
+            "name": "body",
+            "in": "body",
+            "required": true,
+            "schema": {
+              "$ref": "#/definitions/etcdserverpbDowngradeRequest"
+            }
+          }
+        ],
+        "responses": {
+          "200": {
+            "description": "A successful response.",
+            "schema": {
+              "$ref": "#/definitions/etcdserverpbDowngradeResponse"
+            }
+          }
+        }
+      }
+    },
     "/v3/maintenance/hash": {
       "post": {
         "tags": [
@@ -1156,6 +1183,15 @@
         "LEASE"
       ]
     },
+    "DowngradeRequestDowngradeAction": {
+      "type": "string",
+      "default": "VALIDATE",
+      "enum": [
+        "VALIDATE",
+        "ENABLE",
+        "CANCEL"
+      ]
+    },
     "EventEventType": {
       "type": "string",
       "default": "PUT",
@@ -1708,6 +1744,31 @@
         }
       }
     },
+    "etcdserverpbDowngradeRequest": {
+      "type": "object",
+      "properties": {
+        "action": {
+          "description": "action is the kind of downgrade request to issue. The action may\nVALIDATE the target version, DOWNGRADE the cluster version,\nor CANCEL the current downgrading job.",
+          "$ref": "#/definitions/DowngradeRequestDowngradeAction"
+        },
+        "version": {
+          "description": "version is the target version to downgrade.",
+          "type": "string"
+        }
+      }
+    },
+    "etcdserverpbDowngradeResponse": {
+      "type": "object",
+      "properties": {
+        "header": {
+          "$ref": "#/definitions/etcdserverpbResponseHeader"
+        },
+        "version": {
+          "description": "version is the current cluster version.",
+          "type": "string"
+        }
+      }
+    },
     "etcdserverpbHashKVRequest": {
       "type": "object",
       "properties": {

clientv3/integration/maintenance_test.go

@@ -25,6 +25,8 @@ import (
 	"testing"
 	"time"
 
+	"github.com/coreos/go-semver/semver"
+	"go.etcd.io/etcd/version"
 	"go.uber.org/zap"
 	"google.golang.org/grpc"
 
@@ -240,3 +242,107 @@ func TestMaintenanceStatus(t *testing.T) {
 		t.Fatal("no leader found")
 	}
 }
+
+func TestDowngradeValidateFromClient(t *testing.T) {
+	defer testutil.AfterTest(t)
+
+	clus := integration.NewClusterV3(t, &integration.ClusterConfig{Size: 3})
+	defer clus.Terminate(t)
+
+	clus.WaitLeader(t)
+
+	eps := make([]string, 3)
+	for i := 0; i < 3; i++ {
+		eps[i] = clus.Members[i].GRPCAddr()
+	}
+
+	cli, err := clientv3.New(clientv3.Config{Endpoints: eps, DialOptions: []grpc.DialOption{grpc.WithBlock()}})
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer cli.Close()
+
+	serverVersion := semver.Must(semver.NewVersion(version.Version))
+	clusterVersion := semver.Version{Major: serverVersion.Major, Minor: serverVersion.Minor}
+	targetVersion := &semver.Version{Major: serverVersion.Major, Minor: serverVersion.Minor - 1}
+	resp, err := cli.DowngradeValidate(context.TODO(), targetVersion.String())
+
+	if err != nil {
+		t.Fatalf("failed to validate downgrade against target version (%v)", err)
+	}
+	if resp.Version != clusterVersion.String() {
+		t.Errorf("expected %v; got %v", clusterVersion.String(), resp.Version)
+	}
+}
+
+func TestDowngradeEnableFromClient(t *testing.T) {
+	defer testutil.AfterTest(t)
+
+	clus := integration.NewClusterV3(t, &integration.ClusterConfig{Size: 3})
+	defer clus.Terminate(t)
+
+	clus.WaitLeader(t)
+
+	eps := make([]string, 3)
+	for i := 0; i < 3; i++ {
+		eps[i] = clus.Members[i].GRPCAddr()
+	}
+
+	cli, err := clientv3.New(clientv3.Config{Endpoints: eps, DialOptions: []grpc.DialOption{grpc.WithBlock()}})
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer cli.Close()
+
+	serverVersion := semver.Must(semver.NewVersion(version.Version))
+	clusterVersion := semver.Version{Major: serverVersion.Major, Minor: serverVersion.Minor}
+	targetVersion := &semver.Version{Major: serverVersion.Major, Minor: serverVersion.Minor - 1}
+	resp, err := cli.DowngradeEnable(context.TODO(), targetVersion.String())
+
+	if err != nil {
+		t.Fatalf("failed to enable downgrade to target version (%v)", err)
+	}
+	if resp.Version != clusterVersion.String() {
+		t.Errorf("expected %v; got %v", clusterVersion.String(), resp.Version)
+	}
+}
+
+func TestDowngradeCancelFromClient(t *testing.T) {
+	defer testutil.AfterTest(t)
+
+	clus := integration.NewClusterV3(t, &integration.ClusterConfig{Size: 3})
+	defer clus.Terminate(t)
+
+	clus.WaitLeader(t)
+
+	eps := make([]string, 3)
+	for i := 0; i < 3; i++ {
+		eps[i] = clus.Members[i].GRPCAddr()
+	}
+
+	cli, err := clientv3.New(clientv3.Config{Endpoints: eps, DialOptions: []grpc.DialOption{grpc.WithBlock()}})
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer cli.Close()
+
+	serverVersion := semver.Must(semver.NewVersion(version.Version))
+	clusterVersion := semver.Version{Major: serverVersion.Major, Minor: serverVersion.Minor}
+	targetVersion := &semver.Version{Major: serverVersion.Major, Minor: serverVersion.Minor - 1}
+	// send downgrade enable request
+	resp, err := cli.DowngradeEnable(context.TODO(), targetVersion.String())
+
+	if err != nil {
+		t.Fatalf("failed to enable downgrade to target version (%v)", err)
+	}
+
+	// send downgrade cancel request
+	resp, err = cli.DowngradeCancel(context.TODO())
+	if err != nil {
+		t.Fatalf("failed to cancel downgrade (%v)", err)
+	}
+
+	if resp.Version != clusterVersion.String() {
+		t.Errorf("expected %v; got %v", clusterVersion.String(), resp.Version)
+	}
+}

clientv3/maintenance.go

@@ -31,6 +31,7 @@ type (
 	StatusResponse     pb.StatusResponse
 	HashKVResponse     pb.HashKVResponse
 	MoveLeaderResponse pb.MoveLeaderResponse
+	DowngradeResponse  pb.DowngradeResponse
 )
 
 type Maintenance interface {
@@ -65,6 +66,23 @@ type Maintenance interface {
 	// MoveLeader requests current leader to transfer its leadership to the transferee.
 	// Request must be made to the leader.
 	MoveLeader(ctx context.Context, transfereeID uint64) (*MoveLeaderResponse, error)
+
+	// DowngradeValidate requests validation of the downgrade request against the target version.
+	// Version should follow the version format "Major.Minor.Patch"(e.g. 3.4.0).
+	// The cluster can only be downgraded to one minor version lower.
+	// All other input version will fail the validation.
+	DowngradeValidate(ctx context.Context, version string) (*DowngradeResponse, error)
+
+	// DowngradeEnable requests to downgrade the current cluster version to target version.
+	// It will first validate the target version.
+	// After all servers have been downgraded to target version,
+	// the downgrade will reset disabled automatically.
+	// Redundant DowngradeEnable will error out.
+	DowngradeEnable(ctx context.Context, version string) (*DowngradeResponse, error)
+
+	// DowngradeCancel cancels the current downgrade job.
+	// If there is no current downgrade job, the request will return error message.
+	DowngradeCancel(ctx context.Context) (*DowngradeResponse, error)
 }
 
 type maintenance struct {
@@ -214,6 +232,47 @@ func (m *maintenance) Snapshot(ctx context.Context) (io.ReadCloser, error) {
 	return &snapshotReadCloser{ctx: ctx, ReadCloser: pr}, nil
 }
 
+func (m *maintenance) DowngradeValidate(ctx context.Context, version string) (*DowngradeResponse, error) {
+	req := &pb.DowngradeRequest{
+		Action:  pb.DowngradeRequest_VALIDATE,
+		Version: version,
+	}
+
+	resp, err := m.remote.Downgrade(ctx, req)
+	if err != nil {
+		return nil, toErr(ctx, err)
+	}
+
+	return (*DowngradeResponse)(resp), nil
+}
+
+func (m *maintenance) DowngradeEnable(ctx context.Context, version string) (*DowngradeResponse, error) {
+	req := &pb.DowngradeRequest{
+		Action:  pb.DowngradeRequest_ENABLE,
+		Version: version,
+	}
+
+	resp, err := m.remote.Downgrade(ctx, req)
+	if err != nil {
+		return nil, toErr(ctx, err)
+	}
+
+	return (*DowngradeResponse)(resp), nil
+}
+
+func (m *maintenance) DowngradeCancel(ctx context.Context) (*DowngradeResponse, error) {
+	req := &pb.DowngradeRequest{
+		Action: pb.DowngradeRequest_CANCEL,
+	}
+
+	resp, err := m.remote.Downgrade(ctx, req)
+	if err != nil {
+		return nil, toErr(ctx, err)
+	}
+
+	return (*DowngradeResponse)(resp), nil
+}
+
 type snapshotReadCloser struct {
 	ctx context.Context
 	io.ReadCloser

clientv3/retry.go

@@ -222,6 +222,10 @@ func (rmc *retryMaintenanceClient) Defragment(ctx context.Context, in *pb.Defrag
 	return rmc.mc.Defragment(ctx, in, opts...)
 }
 
+func (rmc *retryMaintenanceClient) Downgrade(ctx context.Context, in *pb.DowngradeRequest, opts ...grpc.CallOption) (resp *pb.DowngradeResponse, err error) {
+	return rmc.mc.Downgrade(ctx, in, opts...)
+}
+
 type retryAuthClient struct {
 	ac pb.AuthClient
 }