etcdserver: support update cluster version through raft

1. Persist the cluster version change through raft. When the member is restarted, it can recover
the previous known decided cluster version.

2. When there is a new leader, it is forced to do a version checking immediately. This helps to
update the first cluster version fast.

etcdserver/cluster.go

@@ -26,6 +26,7 @@ import (
 	"strings"
 	"sync"
 
+	"github.com/coreos/etcd/Godeps/_workspace/src/github.com/coreos/go-semver/semver"
 	"github.com/coreos/etcd/pkg/flags"
 	"github.com/coreos/etcd/pkg/netutil"
 	"github.com/coreos/etcd/pkg/types"
@@ -60,7 +61,8 @@ type Cluster struct {
 	token string
 	store store.Store
 
-	sync.Mutex // guards members and removed map
+	sync.Mutex // guards the fields below
+	version    *semver.Version
 	members    map[types.ID]*Member
 	// removed contains the ids of removed members in the cluster.
 	// removed id cannot be reused.
@@ -100,6 +102,7 @@ func NewClusterFromStore(token string, st store.Store) *Cluster {
 	c := newCluster(token)
 	c.store = st
 	c.members, c.removed = membersFromStore(c.store)
+	c.version = clusterVersionFromStore(c.store)
 	return c
 }
 
@@ -232,6 +235,7 @@ func (c *Cluster) SetStore(st store.Store) { c.store = st }
 
 func (c *Cluster) Recover() {
 	c.members, c.removed = membersFromStore(c.store)
+	c.version = clusterVersionFromStore(c.store)
 }
 
 // ValidateConfigurationChange takes a proposed ConfChange and
@@ -347,6 +351,26 @@ func (c *Cluster) UpdateRaftAttributes(id types.ID, raftAttr RaftAttributes) {
 	c.members[id].RaftAttributes = raftAttr
 }
 
+func (c *Cluster) Version() *semver.Version {
+	c.Lock()
+	defer c.Unlock()
+	if c.version == nil {
+		return nil
+	}
+	return semver.Must(semver.NewVersion(c.version.String()))
+}
+
+func (c *Cluster) SetVersion(ver *semver.Version) {
+	c.Lock()
+	defer c.Unlock()
+	if c.version != nil {
+		log.Printf("etcdsever: updated the cluster version from %v to %v", c.version.String(), ver.String())
+	} else {
+		log.Printf("etcdsever: set the initial cluster version to %v", ver.String())
+	}
+	c.version = ver
+}
+
 // Validate ensures that there is no identical urls in the cluster peer list
 func (c *Cluster) Validate() error {
 	urlMap := make(map[string]bool)
@@ -392,6 +416,17 @@ func membersFromStore(st store.Store) (map[types.ID]*Member, map[types.ID]bool)
 	return members, removed
 }
 
+func clusterVersionFromStore(st store.Store) *semver.Version {
+	e, err := st.Get(path.Join(StoreClusterPrefix, "version"), false, false)
+	if err != nil {
+		if isKeyNotFound(err) {
+			return nil
+		}
+		log.Panicf("etcdserver: unexpected error (%v) when getting cluster version from store", err)
+	}
+	return semver.Must(semver.NewVersion(*e.Node.Value))
+}
+
 // ValidateClusterAndAssignIDs validates the local cluster by matching the PeerURLs
 // with the existing cluster. If the validation succeeds, it assigns the IDs
 // from the existing cluster to the local cluster.

etcdserver/cluster_test.go

@@ -21,6 +21,7 @@ import (
 	"reflect"
 	"testing"
 
+	"github.com/coreos/etcd/Godeps/_workspace/src/github.com/coreos/go-semver/semver"
 	"github.com/coreos/etcd/pkg/testutil"
 	"github.com/coreos/etcd/pkg/types"
 	"github.com/coreos/etcd/raft/raftpb"
@@ -79,33 +80,48 @@ func TestClusterFromStringBad(t *testing.T) {
 func TestClusterFromStore(t *testing.T) {
 	tests := []struct {
 		mems []*Member
+		ver  *semver.Version
 	}{
 		{
 			[]*Member{newTestMember(1, nil, "", nil)},
+			semver.Must(semver.NewVersion("2.0.0")),
 		},
 		{
 			nil,
+			nil,
 		},
 		{
 			[]*Member{
 				newTestMember(1, nil, "", nil),
 				newTestMember(2, nil, "", nil),
 			},
+			semver.Must(semver.NewVersion("2.0.0")),
 		},
 	}
 	for i, tt := range tests {
+		st := store.New()
 		hc := newTestCluster(nil)
-		hc.SetStore(store.New())
+		hc.SetStore(st)
 		for _, m := range tt.mems {
 			hc.AddMember(m)
 		}
-		c := NewClusterFromStore("abc", hc.store)
+		if tt.ver != nil {
+			_, err := st.Set(path.Join(StoreClusterPrefix, "version"), false, tt.ver.String(), store.Permanent)
+			if err != nil {
+				t.Fatal(err)
+			}
+		}
+
+		c := NewClusterFromStore("abc", st)
 		if c.token != "abc" {
 			t.Errorf("#%d: token = %v, want %v", i, c.token, "abc")
 		}
 		if !reflect.DeepEqual(c.Members(), tt.mems) {
 			t.Errorf("#%d: members = %v, want %v", i, c.Members(), tt.mems)
 		}
+		if !reflect.DeepEqual(c.Version(), tt.ver) {
+			t.Errorf("#%d: ver = %v, want %v", i, c.Version(), tt.ver)
+		}
 	}
 }
 

etcdserver/cluster_util.go

@@ -156,3 +156,34 @@ func decideClusterVersion(vers map[string]string) *semver.Version {
 	}
 	return cv
 }
+
+// getVersion returns the version of the given member via its
+// peerURLs. Returns the last error if it fails to get the version.
+func getVersion(m *Member, tr *http.Transport) (string, error) {
+	cc := &http.Client{
+		Transport: tr,
+		Timeout:   time.Second,
+	}
+	var (
+		err  error
+		resp *http.Response
+	)
+
+	for _, u := range m.PeerURLs {
+		resp, err = cc.Get(u + "/version")
+		if err != nil {
+			continue
+		}
+		b, err := ioutil.ReadAll(resp.Body)
+		resp.Body.Close()
+		if err != nil {
+			continue
+		}
+		var vers version.Versions
+		if err := json.Unmarshal(b, &vers); err != nil {
+			continue
+		}
+		return vers.Server, nil
+	}
+	return "", err
+}

etcdserver/member.go

@@ -19,17 +19,14 @@ import (
 	"encoding/binary"
 	"encoding/json"
 	"fmt"
-	"io/ioutil"
 	"log"
 	"math/rand"
-	"net/http"
 	"path"
 	"sort"
 	"time"
 
 	"github.com/coreos/etcd/pkg/types"
 	"github.com/coreos/etcd/store"
-	"github.com/coreos/etcd/version"
 )
 
 // RaftAttributes represents the raft related attributes of an etcd member.
@@ -152,37 +149,6 @@ func nodeToMember(n *store.NodeExtern) (*Member, error) {
 	return m, nil
 }
 
-// getVersion returns the version of the given member via its
-// peerURLs. Returns the last error if it fails to get the version.
-func getVersion(m *Member, tr *http.Transport) (string, error) {
-	cc := &http.Client{
-		Transport: tr,
-		Timeout:   time.Second,
-	}
-	var (
-		err  error
-		resp *http.Response
-	)
-
-	for _, u := range m.PeerURLs {
-		resp, err = cc.Get(u + "/version")
-		if err != nil {
-			continue
-		}
-		b, err := ioutil.ReadAll(resp.Body)
-		resp.Body.Close()
-		if err != nil {
-			continue
-		}
-		var vers version.Versions
-		if err := json.Unmarshal(b, &vers); err != nil {
-			continue
-		}
-		return vers.Server, nil
-	}
-	return "", err
-}
-
 // implement sort by ID interface
 type SortableMemberSlice []*Member