Improve internal/cluster testing, fix tests

internal/cluster/cluster.go

@@ -5,15 +5,15 @@ import (
 )
 
 type Cluster struct {
-	shards []*Shard
+	shards map[string]*Shard
 }
 
 func New(shards []*mdbstructs.Shard) *Cluster {
 	c := &Cluster{
-		shards: make([]*Shard, 0),
+		shards: make(map[string]*Shard),
 	}
 	for _, shard := range shards {
-		c.shards = append(c.shards, NewShard(shard))
+		c.shards[shard.Id] = NewShard(shard)
 	}
 	return c
 }

internal/cluster/cluster_test.go

@@ -14,10 +14,8 @@ func TestNew(t *testing.T) {
 	if len(cluster.shards) != 1 {
 		t.Fatal("Got unexpected number of shards")
 	}
-	shard := cluster.shards[0]
-	if shard.config.Id != "shard1" {
-		t.Fatal("Got unexpected shard info")
-	} else if len(shard.replset.addrs) != 3 {
+	shard := cluster.shards["shard1"]
+	if len(shard.replset.addrs) != 3 {
 		t.Fatal("Got unexpected replset addresses for shard")
 	}
 }

internal/cluster/replset.go

@@ -1,11 +1,12 @@
 package cluster
 
 import (
+	"sync"
 	"time"
 
 	"github.com/globalsign/mgo"
-	rsConfig "github.com/timvaillancourt/go-mongodb-replset/config"
-	rsStatus "github.com/timvaillancourt/go-mongodb-replset/status"
+	"github.com/globalsign/mgo/bson"
+	"github.com/percona/mongodb-backup/mdbstructs"
 )
 
 const (
@@ -18,34 +19,64 @@ var (
 )
 
 type Replset struct {
-	name  string
-	addrs []string
+	sync.Mutex
+	name     string
+	addrs    []string
+	username string
+	password string
+	session  *mgo.Session
 }
 
-func (r *Replset) GetSession(username, password string) (*mgo.Session, error) {
-	session, err := mgo.DialWithInfo(&mgo.DialInfo{
+func NewReplset(name string, addrs []string, username, password string) (*Replset, error) {
+	r := &Replset{
+		name:     name,
+		addrs:    addrs,
+		username: username,
+		password: password,
+	}
+	return r, r.getSession()
+}
+
+func (r *Replset) getSession() error {
+	r.Lock()
+	defer r.Unlock()
+
+	var err error
+	r.session, err = mgo.DialWithInfo(&mgo.DialInfo{
 		Addrs:          r.addrs,
-		Username:       username,
-		Password:       password,
+		Username:       r.username,
+		Password:       r.password,
 		ReplicaSetName: r.name,
-		FailFast:       true,
 		Timeout:        10 * time.Second,
 	})
-	if err != nil {
-		return nil, err
+	if err != nil || r.session.Ping() != nil {
+		return err
+	}
+	r.session.SetMode(replsetReadPreference, true)
+	return nil
+}
+
+func (r *Replset) Close() {
+	r.Lock()
+	defer r.Unlock()
+
+	if r.session != nil {
+		r.session.Close()
 	}
-	session.SetMode(replsetReadPreference, true)
-	return session, nil
 }
 
-func (r *Replset) GetConfig() (*rsConfig.Config, error) {
-	return nil, nil
+func (r *Replset) GetConfig() (*mdbstructs.ReplsetConfig, error) {
+	rsGetConfig := mdbstructs.ReplSetGetConfig{}
+	err := r.session.Run(bson.D{{"replSetGetConfig", "1"}}, &rsGetConfig)
+	return rsGetConfig.Config, err
 }
 
-func (r *Replset) GetStatus() (*rsStatus.Status, error) {
-	return nil, nil
+func (r *Replset) GetStatus() (*mdbstructs.ReplsetStatus, error) {
+	status := mdbstructs.ReplsetStatus{}
+	err := r.session.Run(bson.D{{"replSetGetStatus", "1"}}, &status)
+	return &status, err
 }
 
-func getBackupNode(config *rsConfig.Config) (*rsConfig.Member, error) {
+func getBackupNode(config *mdbstructs.ReplsetConfig, status *mdbstructs.ReplsetStatus) (*mdbstructs.ReplsetConfigMember, error) {
 	return config.Members[0], nil
 }

internal/cluster/replset_test.go

@@ -0,0 +1,74 @@
+package cluster
+
+import (
+	"testing"
+
+	"github.com/percona/mongodb-backup/internal/testutils"
+)
+
+func TestNewReplset(t *testing.T) {
+	rs, err := NewReplset(
+		testutils.MongoDBReplsetName,
+		[]string{
+			testutils.MongoDBHost + ":" + testutils.MongoDBPrimaryPort,
+		},
+		testutils.MongoDBUser,
+		testutils.MongoDBPassword,
+	)
+	if err != nil {
+		t.Fatalf("Failed to create new replset struct: %v", err.Error())
+	}
+	if rs == nil {
+		rs.Close()
+		t.Fatal("Got nil replset from .NewReplset()")
+	}
+	rs.Close()
+}
+
+func TestGetConfig(t *testing.T) {
+	rs, err := NewReplset(
+		testutils.MongoDBReplsetName,
+		[]string{
+			testutils.MongoDBHost + ":" + testutils.MongoDBPrimaryPort,
+		},
+		testutils.MongoDBUser,
+		testutils.MongoDBPassword,
+	)
+	if err != nil {
+		t.Fatalf("Failed to create new replset struct: %v", err.Error())
+	}
+	defer rs.Close()
+
+	config, err := rs.GetConfig()
+	if err != nil {
+		t.Fatalf("Failed to run .GetConfig() on Replset struct: %v", err.Error())
+	} else if config.Name != testutils.MongoDBReplsetName {
+		t.Fatalf("Got unexpected output from .GetConfig(), expected: %v, got %v", testutils.MongoDBReplsetName, config.Name)
+	} else if len(config.Members) != 3 {
+		t.Fatal("Unexpected number of replica set members in .GetConfig() result")
+	}
+}
+
+func TestGetStatus(t *testing.T) {
+	rs, err := NewReplset(
+		testutils.MongoDBReplsetName,
+		[]string{
+			testutils.MongoDBHost + ":" + testutils.MongoDBPrimaryPort,
+		},
+		testutils.MongoDBUser,
+		testutils.MongoDBPassword,
+	)
+	if err != nil {
+		t.Fatalf("Failed to create new replset struct: %v", err.Error())
+	}
+	defer rs.Close()
+
+	status, err := rs.GetStatus()
+	if err != nil {
+		t.Fatalf("Failed to run .GetStatus() on Replset struct: %v", err.Error())
+	} else if status.Set != testutils.MongoDBReplsetName {
+		t.Fatal("Got unexpected output from .GetStatus()")
+	} else if len(status.Members) != 3 {
+		t.Fatal("Unexpected number of replica set members in .GetStatus() result")
+	}
+}

mdbstructs/replset_config.go

@@ -0,0 +1,61 @@
+package mdbstructs
+
+import "github.com/globalsign/mgo/bson"
+
+// Replica Set tags: https://docs.mongodb.com/manual/tutorial/configure-replica-set-tag-sets/#add-tag-sets-to-a-replica-set
+type ReplsetTags map[string]string
+
+// Write Concern document: https://docs.mongodb.com/manual/reference/write-concern/
+type WriteConcern struct {
+	WriteConcern interface{} `bson:"w" json:"w"`
+	WriteTimeout int         `bson:"wtimeout" json:"wtimeout"`
+	Journal      bool        `bson:"j,omitempty" json:"j,omitempty"`
+}
+
+// Standard MongoDB response
+type OkResponse struct {
+	Ok int `bson:"ok" json:"ok" json:"ok"`
+}
+
+// Member document from 'replSetGetConfig': https://docs.mongodb.com/manual/reference/command/replSetGetConfig/#dbcmd.replSetGetConfig
+type ReplsetConfigMember struct {
+	Id           int          `bson:"_id" json:"_id"`
+	Host         string       `bson:"host" json:"host"`
+	ArbiterOnly  bool         `bson:"arbiterOnly" json:"arbiterOnly"`
+	BuildIndexes bool         `bson:"buildIndexes" json:"buildIndexes"`
+	Hidden       bool         `bson:"hidden" json:"hidden"`
+	Priority     int          `bson:"priority" json:"priority"`
+	Tags         *ReplsetTags `bson:"tags,omitempty" json:"tags,omitempty"`
+	SlaveDelay   int64        `bson:"slaveDelay" json:"slaveDelay"`
+	Votes        int          `bson:"votes" json:"votes"`
+}
+
+// Settings document from 'replSetGetConfig': https://docs.mongodb.com/manual/reference/command/replSetGetConfig/#dbcmd.replSetGetConfig
+type ReplsetConfigSettings struct {
+	ChainingAllowed         bool                    `bson:"chainingAllowed,omitempty" json:"chainingAllowed,omitempty"`
+	HeartbeatIntervalMillis int64                   `bson:"heartbeatIntervalMillis,omitempty" json:"heartbeatIntervalMillis,omitempty"`
+	HeartbeatTimeoutSecs    int                     `bson:"heartbeatTimeoutSecs,omitempty" json:"heartbeatTimeoutSecs,omitempty"`
+	ElectionTimeoutMillis   int64                   `bson:"electionTimeoutMillis,omitempty" json:"electionTimeoutMillis,omitempty"`
+	CatchUpTimeoutMillis    int64                   `bson:"catchUpTimeoutMillis,omitempty" json:"catchUpTimeoutMillis,omitempty"`
+	GetLastErrorModes       map[string]*ReplsetTags `bson:"getLastErrorModes,omitempty" json:"getLastErrorModes,omitempty"`
+	GetLastErrorDefaults    *WriteConcern           `bson:"getLastErrorDefaults,omitempty" json:"getLastErrorDefaults,omitempty"`
+	ReplicaSetId            bson.ObjectId           `bson:"replicaSetId,omitempty" json:"replicaSetId,omitempty"`
+}
+
+// Config document from 'replSetGetConfig': https://docs.mongodb.com/manual/reference/command/replSetGetConfig/#dbcmd.replSetGetConfig
+type ReplsetConfig struct {
+	Name                               string                 `bson:"_id" json:"_id"`
+	Version                            int                    `bson:"version" json:"version"`
+	Members                            []*ReplsetConfigMember `bson:"members" json:"members"`
+	Configsvr                          bool                   `bson:"configsvr,omitempty" json:"configsvr,omitempty"`
+	ProtocolVersion                    int                    `bson:"protocolVersion,omitempty" json:"protocolVersion,omitempty"`
+	Settings                           *ReplsetConfigSettings `bson:"settings,omitempty" json:"settings,omitempty"`
+	WriteConcernMajorityJournalDefault bool                   `bson:"writeConcernMajorityJournalDefault,omitempty" json:"writeConcernMajorityJournalDefault,omitempty"`
+}
+
+// Response document from 'replSetGetConfig': https://docs.mongodb.com/manual/reference/command/replSetGetConfig/#dbcmd.replSetGetConfig
+type ReplSetGetConfig struct {
+	Config *ReplsetConfig `bson:"config" json:"config"`
+	Errmsg string         `bson:"errmsg,omitempty" json:"errmsg,omitempty"`
+	Ok     int            `bson:"ok" json:"ok" json:"ok"`
+}

mdbstructs/replset_status.go

@@ -0,0 +1,90 @@
+package mdbstructs
+
+import (
+	"time"
+
+	"github.com/globalsign/mgo/bson"
+)
+
+type ReplsetMemberHealth int
+type ReplsetMemberState int
+
+const (
+	ReplsetMemberHealthDown ReplsetMemberHealth = iota
+	ReplsetMemberHealthUp
+	ReplsetMemberStateStartup    ReplsetMemberState = 0
+	ReplsetMemberStatePrimary    ReplsetMemberState = 1
+	ReplsetMemberStateSecondary  ReplsetMemberState = 2
+	ReplsetMemberStateRecovering ReplsetMemberState = 3
+	ReplsetMemberStateStartup2   ReplsetMemberState = 5
+	ReplsetMemberStateUnknown    ReplsetMemberState = 6
+	ReplsetMemberStateArbiter    ReplsetMemberState = 7
+	ReplsetMemberStateDown       ReplsetMemberState = 8
+	ReplsetMemberStateRollback   ReplsetMemberState = 9
+	ReplsetMemberStateRemoved    ReplsetMemberState = 10
+)
+
+var ReplsetMemberStateStrings = map[ReplsetMemberState]string{
+	ReplsetMemberStateStartup:    "STARTUP",
+	ReplsetMemberStatePrimary:    "PRIMARY",
+	ReplsetMemberStateSecondary:  "SECONDARY",
+	ReplsetMemberStateRecovering: "RECOVERING",
+	ReplsetMemberStateStartup2:   "STARTUP2",
+	ReplsetMemberStateUnknown:    "UNKNOWN",
+	ReplsetMemberStateArbiter:    "ARBITER",
+	ReplsetMemberStateDown:       "DOWN",
+	ReplsetMemberStateRollback:   "ROLLBACK",
+	ReplsetMemberStateRemoved:    "REMOVED",
+}
+
+func (ms ReplsetMemberState) String() string {
+	if str, ok := ReplsetMemberStateStrings[ms]; ok {
+		return str
+	}
+	return ""
+}
+
+type Optime struct {
+	Timestamp bson.MongoTimestamp `bson:"ts" json:"ts"`
+	Term      int64               `bson:"t" json:"t"`
+}
+
+type StatusOptimes struct {
+	LastCommittedOpTime *Optime `bson:"lastCommittedOpTime" json:"lastCommittedOpTime"`
+	AppliedOpTime       *Optime `bson:"appliedOpTime" json:"appliedOpTime"`
+	DurableOptime       *Optime `bson:"durableOpTime" json:"durableOpTime"`
+}
+
+type ReplsetStatusMember struct {
+	Id                int                 `bson:"_id" json:"_id"`
+	Name              string              `bson:"name" json:"name"`
+	Health            ReplsetMemberHealth `bson:"health" json:"health"`
+	State             ReplsetMemberState  `bson:"state" json:"state"`
+	StateStr          string              `bson:"stateStr" json:"stateStr"`
+	Uptime            int64               `bson:"uptime" json:"uptime"`
+	Optime            *Optime             `bson:"optime" json:"optime"`
+	OptimeDate        time.Time           `bson:"optimeDate" json:"optimeDate"`
+	ConfigVersion     int                 `bson:"configVersion" json:"configVersion"`
+	ElectionTime      bson.MongoTimestamp `bson:"electionTime,omitempty" json:"electionTime,omitempty"`
+	ElectionDate      time.Time           `bson:"electionDate,omitempty" json:"electionDate,omitempty"`
+	InfoMessage       string              `bson:"infoMessage,omitempty" json:"infoMessage,omitempty"`
+	OptimeDurable     *Optime             `bson:"optimeDurable,omitempty" json:"optimeDurable,omitempty"`
+	OptimeDurableDate time.Time           `bson:"optimeDurableDate,omitempty" json:"optimeDurableDate,omitempty"`
+	LastHeartbeat     time.Time           `bson:"lastHeartbeat,omitempty" json:"lastHeartbeat,omitempty"`
+	LastHeartbeatRecv time.Time           `bson:"lastHeartbeatRecv,omitempty" json:"lastHeartbeatRecv,omitempty"`
+	PingMs            int64               `bson:"pingMs,omitempty" json:"pingMs,omitempty"`
+	Self              bool                `bson:"self,omitempty" json:"self,omitempty"`
+	SyncingTo         string              `bson:"syncingTo,omitempty" json:"syncingTo,omitempty"`
+}
+
+type ReplsetStatus struct {
+	Set                     string                 `bson:"set" json:"set"`
+	Date                    time.Time              `bson:"date" json:"date"`
+	MyState                 ReplsetMemberState     `bson:"myState" json:"myState"`
+	Members                 []*ReplsetStatusMember `bson:"members" json:"members"`
+	Term                    int64                  `bson:"term,omitempty" json:"term,omitempty"`
+	HeartbeatIntervalMillis int64                  `bson:"heartbeatIntervalMillis,omitempty" json:"heartbeatIntervalMillis,omitempty"`
+	Optimes                 *StatusOptimes         `bson:"optimes,omitempty" json:"optimes,omitempty"`
+	Errmsg                  string                 `bson:"errmsg,omitempty" json:"errmsg,omitempty"`
+	Ok                      int                    `bson:"ok" json:"ok"`
+}

mdbstructs/sharding.go

@@ -2,12 +2,22 @@ package mdbstructs
 
 import "github.com/globalsign/mgo/bson"
 
+// Shard reflects a document in the config server 'config.shards'
+// collection (or the 'shards' array of the 'listShards' server
+// command).
+//
+// https://docs.mongodb.com/manual/reference/config-database/#config.shards
+//
 type Shard struct {
 	Id    string `bson:"_id"`
 	Host  string `bson:"host"`
 	State int    `bson:"state"`
 }
 
+// ListShards reflects the output of the MongoDB 'listShards' command.
+//
+// https://docs.mongodb.com/manual/reference/command/listShards/
+//
 type ListShards struct {
 	Shards        []*Shard            `bson:"shards"`
 	Ok            int                 `bson:"ok"`