Fix broken tests from adding internal/cluster

Dockerfile.test

@@ -4,5 +4,6 @@ FROM golang:$GOLANG_DOCKERHUB_TAG
 WORKDIR /go/src/github.com/percona/mongodb-backup
 COPY . .
 
-RUN go get ./...
+RUN go get ./... && \
+    go vet ./...
 CMD make test-race

Makefile

@@ -22,16 +22,28 @@ AWS_SECRET_ACCESS_KEY?=
 
 all: test
 
-test:
-	GOCACHE=$(GOCACHE) go test -v $(GO_TEST_PATH)
-
 test-race:
 ifeq ($(GO_TEST_CODECOV), true)
 	GOCACHE=$(GOCACHE) go test -v -race -coverprofile=$(GO_TEST_COVER_PROFILE) -covermode=atomic $(GO_TEST_PATH)
 else
 	GOCACHE=$(GOCACHE) go test -v -race $(GO_TEST_PATH)
 endif
 
+test:
+	TEST_MONGODB_ADMIN_USERNAME=$(TEST_MONGODB_ADMIN_USERNAME) \
+	TEST_MONGODB_ADMIN_PASSWORD=$(TEST_MONGODB_ADMIN_PASSWORD) \
+	TEST_MONGODB_USERNAME=$(TEST_MONGODB_USERNAME) \
+	TEST_MONGODB_PASSWORD=$(TEST_MONGODB_PASSWORD) \
+	TEST_MONGODB_RS=$(TEST_MONGODB_RS) \
+	TEST_MONGODB_PRIMARY_PORT=$(TEST_MONGODB_PRIMARY_PORT) \
+	TEST_MONGODB_SECONDARY1_PORT=$(TEST_MONGODB_SECONDARY1_PORT) \
+	TEST_MONGODB_SECONDARY2_PORT=$(TEST_MONGODB_SECONDARY2_PORT) \
+	TEST_MONGODB_CONFIGSVR_RS=$(TEST_MONGODB_CONFIGSVR_RS) \
+	TEST_MONGODB_CONFIGSVR1_PORT=$(TEST_MONGODB_CONFIGSVR1_PORT) \
+	TEST_MONGODB_MONGOS_PORT=$(TEST_MONGODB_MONGOS_PORT) \
+	GOCACHE=$(GOCACHE) \
+	go test -v $(GO_TEST_PATH)
+
 test-cluster:
 	TEST_PSMDB_VERSION=$(TEST_PSMDB_VERSION) \
 	TEST_MONGODB_ADMIN_USERNAME=$(TEST_MONGODB_ADMIN_USERNAME) \

docker-compose.yml

@@ -35,7 +35,7 @@ services:
     network_mode: host
     image: percona/percona-server-mongodb:${TEST_PSMDB_VERSION:-latest}
     entrypoint: /entrypoint-mongod.sh
-    command: --replSet=csReplSet --port=${TEST_MONGODB_CONFIGSVR1_PORT:-17004} --configsvr
+    command: --replSet=${TEST_MONGODB_CONFIGSVR_RS:-csReplSet} --port=${TEST_MONGODB_CONFIGSVR1_PORT:-17004} --configsvr
     volumes:
     - ./scripts/entrypoint-mongod.sh:/entrypoint-mongod.sh:ro
     - ./scripts/mongod.key:/mongod.key:ro
@@ -101,6 +101,8 @@ services:
     - TEST_MONGODB_MONGOS_PORT=${TEST_MONGODB_MONGOS_PORT:-17005}
     volumes:
     - ./test-out:/tmp/out
+    - ./scripts/ssl/rootCA.crt:/rootCA.crt:ro
+    - ./scripts/ssl/client.pem:/client.pem:ro
     depends_on:
     - configsvr
     - mongo1

internal/cluster/cluster.go

@@ -1,45 +1,19 @@
 package cluster
 
 import (
-	"strings"
-
-	"github.com/globalsign/mgo"
-	"github.com/globalsign/mgo/bson"
 	"github.com/percona/mongodb-backup/mdbstructs"
 )
 
-type Shard struct {
-	config  *mdbstructs.ListShardsShard
-	replset *Replset
+type Cluster struct {
+	shards []*Shard
 }
 
-func parseShardURI(uri string) (string, []string) {
-	s := strings.Split(uri, "/")
-	if len(s) == 2 {
-		return s[0], strings.Split(s[1], ",")
+func New(shards []*mdbstructs.Shard) *Cluster {
+	c := &Cluster{
+		shards: make([]*Shard, 0),
 	}
-	return "", []string{}
-}
-
-func NewShard(shard *mdbstructs.ListShardsShard) *Shard {
-	replset, addrs := parseShardURI(shard.Host)
-	return &Shard{
-		config: shard,
-		replset: &Replset{
-			name:  replset,
-			addrs: addrs,
-		},
+	for _, shard := range shards {
+		c.shards = append(c.shards, NewShard(shard))
 	}
-}
-
-// Return the shards within a sharded cluster using the MongoDB 'listShards'
-// server command. This command will only succeed on a mongos or config
-// server.
-//
-// https://docs.mongodb.com/manual/reference/command/listShards/
-//
-func GetListShards(session *mgo.Session) (*mdbstructs.ListShards, error) {
-	listShards := mdbstructs.ListShards{}
-	err := session.Run(bson.D{{"listShards", "1"}}, &listShards)
-	return &listShards, err
+	return c
 }

internal/cluster/cluster_test.go

@@ -1,64 +1,23 @@
 package cluster
 
 import (
-	"reflect"
 	"testing"
 
-	"github.com/globalsign/mgo"
-	"github.com/percona/mongodb-backup/internal/testutils"
 	"github.com/percona/mongodb-backup/mdbstructs"
 )
 
-func TestParseShardURI(t *testing.T) {
-	rs, hosts := parseShardURI("rs/127.0.0.1:27017,127.0.0.1:27018,127.0.0.1:27019")
-	if rs == "" {
-		t.Fatal("Got empty replset name from .parseShardURI()")
-	} else if len(hosts) != 3 {
-		t.Fatalf("Expected %d hosts but got %d from .parseShardURI()", 3, len(hosts))
-	} else if rs != "rs" {
-		t.Fatalf("Expected replset name %s but got %s from .parseShardURI()", "rs", rs)
-	} else if !reflect.DeepEqual(hosts, []string{"127.0.0.1:27017", "127.0.0.1:27018", "127.0.0.1:27019"}) {
-		t.Fatal("Unexpected hosts list")
-	}
-
-	// too many '/'
-	rs, hosts = parseShardURI("rs///")
-	if rs != "" || len(hosts) > 0 {
-		t.Fatal("Expected empty results from .parseShardURI()")
-	}
-
-	// missing replset
-	rs, hosts = parseShardURI("127.0.0.1:27017")
-	if rs != "" || len(hosts) > 0 {
-		t.Fatal("Expected empty results from .parseShardURI()")
-	}
-}
-
-func TestNewShard(t *testing.T) {
-	shard := NewShard(&mdbstructs.ListShardShard{
+func TestNew(t *testing.T) {
+	cluster := New([]*mdbstructs.Shard{{
 		Id:   "shard1",
 		Host: "rs/127.0.0.1:27017,127.0.0.1:27018,127.0.0.1:27019",
-	})
-	if shard.replset.name != "rs" {
-		t.Fatalf("Expected 'replset.name' to equal %v but got %v", "rs", shard.replset.name)
-	} else if len(shard.replset.addrs) != 3 {
-		t.Fatalf("Expected 'replset.addrs' to contain %d addresses but got %d", 3, len(shard.replset.addrs))
-	}
-}
-
-func TestGetListShards(t *testing.T) {
-	session, err := mgo.DialWithInfo(testutils.MongosDialInfo())
-	if err != nil {
-		t.Fatalf("Failed to connect to mongos: %v", err.Error())
+	}})
+	if len(cluster.shards) != 1 {
+		t.Fatal("Got unexpected number of shards")
 	}
-	defer session.Close()
-
-	listShards, err := GetListShards(session)
-	if err != nil {
-		t.Fatalf("Failed to run 'listShards' command: %v", err.Error())
-	} else if listShards.Ok != 1 {
-		t.Fatal("Got non-ok response code from 'listShards' command")
-	} else if len(listShards.Shards) < 1 {
-		t.Fatal("Got zero shards from .GetListShards()")
+	shard := cluster.shards[0]
+	if shard.config.Id != "shard1" {
+		t.Fatal("Got unexpected shard info")
+	} else if len(shard.replset.addrs) != 3 {
+		t.Fatal("Got unexpected replset addresses for shard")
 	}
 }

internal/cluster/is_master_test.go

@@ -31,6 +31,22 @@ func TestIsReplset(t *testing.T) {
 	if isReplset(&mdbstructs.IsMaster{SetName: ""}) {
 		t.Fatal("Expected false from .isReplset()")
 	}
+
+	session, err := mgo.DialWithInfo(testutils.PrimaryDialInfo())
+	if err != nil {
+		t.Fatalf("Could not connect to primary: %v", err.Error())
+	}
+	defer session.Close()
+
+	isMaster, err := GetIsMaster(session)
+	if err != nil {
+		session.Close()
+		t.Fatalf("Could not run 'isMaster' command: %v", err.Error())
+	}
+	if !isReplset(isMaster) {
+		session.Close()
+		t.Fatalf("Expected true from .isReplset()")
+	}
 }
 
 func TestIsMongos(t *testing.T) {
@@ -43,6 +59,22 @@ func TestIsMongos(t *testing.T) {
 	if isMongos(&mdbstructs.IsMaster{IsMaster: true}) {
 		t.Fatal("Expected false from .isMongos()")
 	}
+
+	session, err := mgo.DialWithInfo(testutils.MongosDialInfo())
+	if err != nil {
+		t.Fatalf("Could not connect to mongos: %v", err.Error())
+	}
+	defer session.Close()
+
+	isMaster, err := GetIsMaster(session)
+	if err != nil {
+		session.Close()
+		t.Fatalf("Could not run 'isMaster' command: %v", err.Error())
+	}
+	if !isMongos(isMaster) {
+		session.Close()
+		t.Fatalf("Expected true from .isMongos()")
+	}
 }
 
 func TestIsConfigServer(t *testing.T) {
@@ -55,6 +87,22 @@ func TestIsConfigServer(t *testing.T) {
 	if isConfigServer(&mdbstructs.IsMaster{SetName: "csReplSet"}) {
 		t.Fatal("Expected true from .isConfigServer()")
 	}
+
+	session, err := mgo.DialWithInfo(testutils.ConfigsvrReplsetDialInfo())
+	if err != nil {
+		t.Fatalf("Could not connect to configsvr replset: %v", err.Error())
+	}
+	defer session.Close()
+
+	isMaster, err := GetIsMaster(session)
+	if err != nil {
+		session.Close()
+		t.Fatalf("Could not run 'isMaster' command: %v", err.Error())
+	}
+	if !isConfigServer(isMaster) {
+		session.Close()
+		t.Fatalf("Expected true from .isConfigServer()")
+	}
 }
 
 func TestIsShardedCluster(t *testing.T) {
@@ -76,4 +124,20 @@ func TestIsShardedCluster(t *testing.T) {
 	}) {
 		t.Fatal("Expected true false isShardedCluster()")
 	}
+
+	session, err := mgo.DialWithInfo(testutils.ConfigsvrReplsetDialInfo())
+	if err != nil {
+		t.Fatalf("Could not connect to configsvr replset: %v", err.Error())
+	}
+	defer session.Close()
+
+	isMaster, err := GetIsMaster(session)
+	if err != nil {
+		session.Close()
+		t.Fatalf("Could not run 'isMaster' command: %v", err.Error())
+	}
+	if !isShardedCluster(isMaster) {
+		session.Close()
+		t.Fatalf("Expected true from .isConfigServer()")
+	}
 }

internal/cluster/shard.go

@@ -0,0 +1,65 @@
+package cluster
+
+import (
+	"strings"
+
+	"github.com/globalsign/mgo"
+	"github.com/globalsign/mgo/bson"
+	"github.com/percona/mongodb-backup/mdbstructs"
+)
+
+const (
+	configDB = "config"
+)
+
+type Shard struct {
+	config  *mdbstructs.Shard
+	replset *Replset
+}
+
+// parseShardURI takes in a MongoDB shard URI (in
+// '<replica-set>/<host1>:<port1>,<host2>:<port2>,...'
+// format) and returns a replica set and slice of hosts.
+func parseShardURI(uri string) (string, []string) {
+	s := strings.Split(uri, "/")
+	if len(s) == 2 {
+		return s[0], strings.Split(s[1], ",")
+	}
+	return "", []string{}
+}
+
+func NewShard(shard *mdbstructs.Shard) *Shard {
+	replset, addrs := parseShardURI(shard.Host)
+	return &Shard{
+		config: shard,
+		replset: &Replset{
+			name:  replset,
+			addrs: addrs,
+		},
+	}
+}
+
+// Return shards within a sharded cluster using the MongoDB 'listShards'
+// server command. This command will only succeed on a mongos or config
+// server.
+//
+// https://docs.mongodb.com/manual/reference/command/listShards/
+//
+func GetListShards(session *mgo.Session) (*mdbstructs.ListShards, error) {
+	listShards := mdbstructs.ListShards{}
+	err := session.Run(bson.D{{"listShards", "1"}}, &listShards)
+	return &listShards, err
+}
+
+// Return shards within a sharded cluster using the 'config.shards'
+// collection on a config server. This is needed because config servers
+// do not have the 'listShards' command. Use .GetListShards() to query
+// shards from a mongos.
+//
+// https://docs.mongodb.com/manual/reference/config-database/#config.shards
+//
+func GetConfigsvrShards(session *mgo.Session) ([]*mdbstructs.Shard, error) {
+	shards := []*mdbstructs.Shard{}
+	err := session.DB(configDB).C("shards").Find(nil).All(&shards)
+	return shards, err
+}