Benchmark concurrent Cassandra LWTs (#6186)

codecov.yml

@@ -52,6 +52,7 @@ ignore:
   - "common/domain/errors.go"
   - "common/log/**"
   - "common/metrics/**"
+  - "common/persistence/nosql/nosqlplugin/cassandra/admin.go"
   - "common/persistence/nosql/nosqlplugin/dynamodb/**"
   - "common/persistence/nosql/nosqlplugin/mongodb/**"
   - "common/types/shared.go" # 8k lines of getters. Not worth testing manually but consider switching to generated code.

common/persistence/nosql/nosql_test_utils.go

@@ -38,6 +38,7 @@ type testCluster struct {
 
 	keyspace      string
 	schemaBaseDir string
+	replicas      int
 	cfg           config.NoSQL
 }
 
@@ -51,6 +52,10 @@ type TestClusterParams struct {
 	Port          int
 	ProtoVersion  int
 	SchemaBaseDir string
+	// Replicas defaults to 1 if not set
+	Replicas int
+	// MaxConns defaults to 2 if not set
+	MaxConns int
 }
 
 // NewTestCluster returns a new cassandra test cluster
@@ -61,13 +66,14 @@ func NewTestCluster(t *testing.T, params TestClusterParams) testcluster.Persiste
 		logger:        testlogger.New(t),
 		keyspace:      params.KeySpace,
 		schemaBaseDir: params.SchemaBaseDir,
+		replicas:      replicas(params.Replicas),
 		cfg: config.NoSQL{
 			PluginName:   params.PluginName,
 			User:         params.Username,
 			Password:     params.Password,
 			Hosts:        params.Host,
 			Port:         params.Port,
-			MaxConns:     2,
+			MaxConns:     maxConns(params.MaxConns),
 			Keyspace:     params.KeySpace,
 			ProtoVersion: params.ProtoVersion,
 		},
@@ -94,7 +100,7 @@ func (s *testCluster) SetupTestDatabase() {
 	if err != nil {
 		s.logger.Fatal(err.Error())
 	}
-	err = adminDB.SetupTestDatabase(s.schemaBaseDir)
+	err = adminDB.SetupTestDatabase(s.schemaBaseDir, s.replicas)
 	if err != nil {
 		s.logger.Fatal(err.Error())
 	}
@@ -111,3 +117,19 @@ func (s *testCluster) TearDownTestDatabase() {
 		s.logger.Fatal(err.Error())
 	}
 }
+
+func replicas(replicas int) int {
+	if replicas == 0 {
+		return 1
+	}
+
+	return replicas
+}
+
+func maxConns(maxConns int) int {
+	if maxConns == 0 {
+		return 2
+	}
+
+	return maxConns
+}

common/persistence/nosql/nosqlplugin/cassandra/admin.go

@@ -41,8 +41,8 @@ const (
 
 var _ nosqlplugin.AdminDB = (*cdb)(nil)
 
-func (db *cdb) SetupTestDatabase(schemaBaseDir string) error {
-	err := db.createCassandraKeyspace(db.session, db.cfg.Keyspace, 1, true)
+func (db *cdb) SetupTestDatabase(schemaBaseDir string, replicas int) error {
+	err := db.createCassandraKeyspace(db.session, db.cfg.Keyspace, replicas, true)
 	if err != nil {
 		return err
 	}

common/persistence/nosql/nosqlplugin/dynamodb/admin.go

@@ -25,7 +25,7 @@ import "github.com/uber/cadence/common/persistence/nosql/nosqlplugin"
 
 var _ nosqlplugin.AdminDB = (*ddb)(nil)
 
-func (db *ddb) SetupTestDatabase(schemaBaseDir string) error {
+func (db *ddb) SetupTestDatabase(schemaBaseDir string, replicas int) error {
 	panic("TODO")
 }
 

common/persistence/nosql/nosqlplugin/interfaces.go

@@ -40,7 +40,7 @@ type (
 
 	// AdminDB is for tooling and testing
 	AdminDB interface {
-		SetupTestDatabase(schemaBaseDir string) error
+		SetupTestDatabase(schemaBaseDir string, replicas int) error
 		TeardownTestDatabase() error
 	}
 

common/persistence/nosql/nosqlplugin/mongodb/admin.go

@@ -36,7 +36,7 @@ const (
 	testSchemaDir = "schema/mongodb/"
 )
 
-func (db *mdb) SetupTestDatabase(schemaBaseDir string) error {
+func (db *mdb) SetupTestDatabase(schemaBaseDir string, replicas int) error {
 	if schemaBaseDir == "" {
 		var err error
 		schemaBaseDir, err = nosqlplugin.GetDefaultTestSchemaDir(testSchemaDir)

common/persistence/persistence-tests/persistenceTestBase.go

@@ -70,6 +70,8 @@ type (
 		SchemaDir       string           `yaml:"-"`
 		ClusterMetadata cluster.Metadata `yaml:"-"`
 		ProtoVersion    int              `yaml:"-"`
+		Replicas        int              `yaml:"-"`
+		MaxConns        int              `yaml:"-"`
 	}
 
 	// TestBase wraps the base setup needed to create workflows over persistence layer.
@@ -138,6 +140,8 @@ func NewTestBaseWithNoSQL(t *testing.T, options *TestBaseOptions) *TestBase {
 		Host:         options.DBHost,
 		Port:         options.DBPort,
 		ProtoVersion: options.ProtoVersion,
+		Replicas:     options.Replicas,
+		MaxConns:     options.MaxConns,
 	})
 	metadata := options.ClusterMetadata
 	if metadata.GetCurrentClusterName() == "" {

docker/buildkite/docker-compose-cassandra-lwt.yml

@@ -0,0 +1,86 @@
+version: "3.5"
+
+services:
+  cass1:
+    container_name: cass1
+    hostname: cass1
+    image: cassandra:4.1.1
+    ports:
+      - "9042:9042"
+    environment: &environment    # Declare and save environments variables into "environment"
+      MAX_HEAP_SIZE: 256M
+      HEAP_NEWSIZE: 128M
+      CASSANDRA_SEEDS: "cass1,cass2"    # The first two nodes will be seeds
+      CASSANDRA_CLUSTER_NAME: SolarSystem
+      CASSANDRA_DC: Mars
+      CASSANDRA_RACK: West
+      CASSANDRA_ENDPOINT_SNITCH: GossipingPropertyFileSnitch
+      CASSANDRA_NUM_TOKENS: 128
+    healthcheck:
+      test: ["CMD", "cqlsh", "-u cassandra", "-p cassandra" ,"-e describe keyspaces"]
+      interval: 15s
+      timeout: 30s
+      retries: 10
+    networks:
+      services-network:
+        aliases:
+          - integration-test
+
+  cass2:
+    container_name: cass2
+    hostname: cass2
+    image: cassandra:4.1.1
+    ports:
+      - "9043:9042"
+    environment: *environment # point to "environment" to use the same environment variables as cass1
+    healthcheck:
+      test: ["CMD", "cqlsh", "-u cassandra", "-p cassandra" ,"-e describe keyspaces"]
+      interval: 15s
+      timeout: 30s
+      retries: 10
+    networks:
+      services-network:
+        aliases:
+          - integration-test
+
+  test-cass-lwt:
+    build:
+      context: ../../
+      dockerfile: ./docker/buildkite/Dockerfile
+    command:
+      - /bin/sh
+      - -e
+      - -c
+      - >
+        go test -timeout 180s
+        -run ^TestCassandraLWT$
+        -count 1
+        -v
+        -tags cassandralwt
+        github.com/uber/cadence/host
+        | tee test.log
+    ports:
+      - "7933:7933"
+      - "7934:7934"
+      - "7935:7935"
+      - "7939:7939"
+    environment:
+      # - "CASSANDRA_HOST=cass1"
+      - "CASSANDRA=1"
+      - "CASSANDRA_SEEDS=cass1,cass2"
+    depends_on:
+      cass1:
+        condition: service_healthy
+      cass2:
+        condition: service_healthy
+    volumes:
+      - ../../:/cadence
+    networks:
+      services-network:
+        aliases:
+          - integration-test
+
+networks:
+  services-network:
+    name: services-network
+    driver: bridge