Refactor: arrange and document dynamic config usages in functional tests

tests/activity_api_pause_test.go

@@ -36,7 +36,6 @@ import (
 	sdkclient "go.temporal.io/sdk/client"
 	"go.temporal.io/sdk/temporal"
 	"go.temporal.io/sdk/workflow"
-	"go.temporal.io/server/common/dynamicconfig"
 	"go.temporal.io/server/common/testing/testvars"
 	"go.temporal.io/server/common/util"
 	"go.temporal.io/server/tests/testcore"
@@ -52,15 +51,16 @@ type ActivityApiPauseClientTestSuite struct {
 	activityRetryPolicy *temporal.RetryPolicy
 }
 
-func (s *ActivityApiPauseClientTestSuite) SetupSuite() {
-	s.FunctionalTestSdkSuite.SetupSuite()
-	s.OverrideDynamicConfig(dynamicconfig.ActivityAPIsEnabled, true)
-	s.tv = testvars.New(s.T()).WithTaskQueue(s.TaskQueue()).WithNamespaceName(s.Namespace())
+func TestActivityApiPauseClientTestSuite(t *testing.T) {
+	s := new(ActivityApiPauseClientTestSuite)
+	suite.Run(t, s)
 }
 
 func (s *ActivityApiPauseClientTestSuite) SetupTest() {
 	s.FunctionalTestSdkSuite.SetupTest()
 
+	s.tv = testvars.New(s.T()).WithTaskQueue(s.TaskQueue()).WithNamespaceName(s.Namespace())
+
 	s.initialRetryInterval = 1 * time.Second
 	s.scheduleToCloseTimeout = 30 * time.Minute
 	s.startToCloseTimeout = 15 * time.Minute
@@ -71,11 +71,6 @@ func (s *ActivityApiPauseClientTestSuite) SetupTest() {
 	}
 }
 
-func TestActivityApiPauseClientTestSuite(t *testing.T) {
-	s := new(ActivityApiPauseClientTestSuite)
-	suite.Run(t, s)
-}
-
 func (s *ActivityApiPauseClientTestSuite) makeWorkflowFunc(activityFunction ActivityFunctions) WorkflowFunction {
 	return func(ctx workflow.Context) error {
 

tests/activity_api_reset_test.go

@@ -38,7 +38,6 @@ import (
 	sdkclient "go.temporal.io/sdk/client"
 	"go.temporal.io/sdk/temporal"
 	"go.temporal.io/sdk/workflow"
-	"go.temporal.io/server/common/dynamicconfig"
 	"go.temporal.io/server/common/testing/testvars"
 	"go.temporal.io/server/common/util"
 	"go.temporal.io/server/tests/testcore"
@@ -54,15 +53,16 @@ type ActivityApiResetClientTestSuite struct {
 	activityRetryPolicy *temporal.RetryPolicy
 }
 
-func (s *ActivityApiResetClientTestSuite) SetupSuite() {
-	s.FunctionalTestSdkSuite.SetupSuite()
-	s.OverrideDynamicConfig(dynamicconfig.ActivityAPIsEnabled, true)
-	s.tv = testvars.New(s.T()).WithTaskQueue(s.TaskQueue()).WithNamespaceName(s.Namespace())
+func TestActivityApiResetClientTestSuite(t *testing.T) {
+	s := new(ActivityApiResetClientTestSuite)
+	suite.Run(t, s)
 }
 
 func (s *ActivityApiResetClientTestSuite) SetupTest() {
 	s.FunctionalTestSdkSuite.SetupTest()
 
+	s.tv = testvars.New(s.T()).WithTaskQueue(s.TaskQueue()).WithNamespaceName(s.Namespace())
+
 	s.initialRetryInterval = 1 * time.Second
 	s.scheduleToCloseTimeout = 30 * time.Minute
 	s.startToCloseTimeout = 15 * time.Minute
@@ -73,11 +73,6 @@ func (s *ActivityApiResetClientTestSuite) SetupTest() {
 	}
 }
 
-func TestActivityApiResetClientTestSuite(t *testing.T) {
-	s := new(ActivityApiResetClientTestSuite)
-	suite.Run(t, s)
-}
-
 func (s *ActivityApiResetClientTestSuite) makeWorkflowFunc(activityFunction ActivityFunctions) WorkflowFunction {
 	return func(ctx workflow.Context) error {
 

tests/activity_api_update_test.go

@@ -40,7 +40,6 @@ import (
 	sdkclient "go.temporal.io/sdk/client"
 	"go.temporal.io/sdk/temporal"
 	"go.temporal.io/sdk/workflow"
-	"go.temporal.io/server/common/dynamicconfig"
 	"go.temporal.io/server/common/testing/testvars"
 	"go.temporal.io/server/tests/testcore"
 	"google.golang.org/protobuf/types/known/durationpb"
@@ -52,19 +51,14 @@ type ActivityApiUpdateClientTestSuite struct {
 	tv *testvars.TestVars
 }
 
-func (s *ActivityApiUpdateClientTestSuite) SetupSuite() {
-	s.FunctionalTestSdkSuite.SetupSuite()
-	s.OverrideDynamicConfig(dynamicconfig.ActivityAPIsEnabled, true)
-	s.tv = testvars.New(s.T()).WithTaskQueue(s.TaskQueue()).WithNamespaceName(s.Namespace())
+func TestActivityApiUpdateClientTestSuite(t *testing.T) {
+	s := new(ActivityApiUpdateClientTestSuite)
+	suite.Run(t, s)
 }
 
 func (s *ActivityApiUpdateClientTestSuite) SetupTest() {
 	s.FunctionalTestSdkSuite.SetupTest()
-}
-
-func TestActivityApiUpdateClientTestSuite(t *testing.T) {
-	s := new(ActivityApiUpdateClientTestSuite)
-	suite.Run(t, s)
+	s.tv = testvars.New(s.T()).WithTaskQueue(s.TaskQueue()).WithNamespaceName(s.Namespace())
 }
 
 type (

tests/archival_test.go

@@ -81,9 +81,10 @@ func TestArchivalSuite(t *testing.T) {
 }
 
 func (s *ArchivalSuite) SetupSuite() {
-	s.FunctionalTestSuite.SetupSuiteWithDefaultCluster(testcore.WithDynamicConfigOverrides(map[dynamicconfig.Key]any{
+	dynamicConfigOverrides := map[dynamicconfig.Key]any{
 		dynamicconfig.ArchivalProcessorArchiveDelay.Key(): time.Duration(0),
-	}))
+	}
+	s.FunctionalTestSuite.SetupSuiteWithDefaultCluster(testcore.WithDynamicConfigOverrides(dynamicConfigOverrides))
 }
 
 func (s *ArchivalSuite) TestArchival_TimerQueueProcessor() {

tests/deployment_test.go

@@ -81,7 +81,7 @@ func TestDeploymentSuite(t *testing.T) {
 }
 
 func (s *DeploymentSuite) SetupSuite() {
-	s.FunctionalTestSuite.SetupSuiteWithDefaultCluster(testcore.WithDynamicConfigOverrides(map[dynamicconfig.Key]any{
+	dynamicConfigOverrides := map[dynamicconfig.Key]any{
 		dynamicconfig.EnableDeployments.Key():                          true,
 		dynamicconfig.FrontendEnableWorkerVersioningDataAPIs.Key():     true,
 		dynamicconfig.FrontendEnableWorkerVersioningWorkflowAPIs.Key(): true,
@@ -99,7 +99,9 @@ func (s *DeploymentSuite) SetupSuite() {
 
 		// Reduce the chance of hitting max batch job limit in tests
 		dynamicconfig.FrontendMaxConcurrentBatchOperationPerNamespace.Key(): maxConcurrentBatchOps,
-	}))
+	}
+
+	s.FunctionalTestSuite.SetupSuiteWithDefaultCluster(testcore.WithDynamicConfigOverrides(dynamicConfigOverrides))
 }
 
 func (s *DeploymentSuite) SetupTest() {

tests/http_api_test.go

@@ -309,7 +309,7 @@ func (s *HttpApiTestSuite) runHTTPAPIBasicsTest_Shorthand(contentType string, pr
 }
 
 func (s *HttpApiTestSuite) TestHTTPHostValidation() {
-	s.GetTestCluster().OverrideDynamicConfig(s.T(), dynamicconfig.FrontendHTTPAllowedHosts, []string{"allowed"})
+	s.OverrideDynamicConfig(dynamicconfig.FrontendHTTPAllowedHosts, []string{"allowed"})
 	{
 		req, err := http.NewRequest("GET", "/system-info", nil)
 		s.Require().NoError(err)

tests/namespace_delete_test.go

@@ -70,6 +70,9 @@ func (s *namespaceTestSuite) SetupSuite() {
 	dynamicConfigOverrides := map[dynamicconfig.Key]any{
 		// Run tests at full speed.
 		dynamicconfig.DeleteNamespaceDeleteActivityRPS.Key(): 1000000,
+
+		dynamicconfig.TransferProcessorUpdateAckInterval.Key():   1 * time.Second,
+		dynamicconfig.VisibilityProcessorUpdateAckInterval.Key(): 1 * time.Second,
 	}
 
 	if testcore.UsingSQLAdvancedVisibility() {
@@ -82,9 +85,9 @@ func (s *namespaceTestSuite) SetupSuite() {
 
 func (s *namespaceTestSuite) Test_NamespaceDelete_InvalidUTF8() {
 	// don't fail for this test, we're testing this behavior specifically
-	s.GetTestCluster().OverrideDynamicConfig(s.T(), dynamicconfig.ValidateUTF8FailRPCRequest, false)
-	s.GetTestCluster().OverrideDynamicConfig(s.T(), dynamicconfig.ValidateUTF8FailRPCResponse, false)
-	s.GetTestCluster().OverrideDynamicConfig(s.T(), dynamicconfig.ValidateUTF8FailPersistence, false)
+	s.OverrideDynamicConfig(dynamicconfig.ValidateUTF8FailRPCRequest, false)
+	s.OverrideDynamicConfig(dynamicconfig.ValidateUTF8FailRPCResponse, false)
+	s.OverrideDynamicConfig(dynamicconfig.ValidateUTF8FailPersistence, false)
 
 	capture := s.GetTestCluster().Host().CaptureMetricsHandler().StartCapture()
 	defer s.GetTestCluster().Host().CaptureMetricsHandler().StopCapture(capture)
@@ -184,7 +187,7 @@ func (s *namespaceTestSuite) Test_NamespaceDelete_OverrideDelay() {
 	ctx, cancel := rpc.NewContextWithTimeoutAndVersionHeaders(10000 * time.Second)
 	defer cancel()
 
-	s.GetTestCluster().OverrideDynamicConfig(s.T(), dynamicconfig.DeleteNamespaceNamespaceDeleteDelay, time.Hour)
+	s.OverrideDynamicConfig(dynamicconfig.DeleteNamespaceNamespaceDeleteDelay, time.Hour)
 
 	retention := 24 * time.Hour
 	_, err := s.FrontendClient().RegisterNamespace(ctx, &workflowservice.RegisterNamespaceRequest{
@@ -522,7 +525,7 @@ func (s *namespaceTestSuite) Test_NamespaceDelete_Protected() {
 	})
 	s.NoError(err)
 
-	s.GetTestCluster().OverrideDynamicConfig(s.T(), dynamicconfig.ProtectedNamespaces, []string{tv.NamespaceName().String()})
+	s.OverrideDynamicConfig(dynamicconfig.ProtectedNamespaces, []string{tv.NamespaceName().String()})
 
 	delResp, err := s.OperatorClient().DeleteNamespace(ctx, &operatorservice.DeleteNamespaceRequest{
 		Namespace: tv.NamespaceName().String(),

tests/sizelimit_test.go

@@ -42,6 +42,7 @@ import (
 	"go.temporal.io/api/workflowservice/v1"
 	"go.temporal.io/server/common"
 	"go.temporal.io/server/common/convert"
+	"go.temporal.io/server/common/dynamicconfig"
 	"go.temporal.io/server/common/log/tag"
 	"go.temporal.io/server/common/payloads"
 	"go.temporal.io/server/service/history/consts"
@@ -59,9 +60,18 @@ func TestSizeLimitFunctionalSuite(t *testing.T) {
 	suite.Run(t, new(SizeLimitFunctionalSuite))
 }
 
-// This cluster use customized threshold for history config
 func (s *SizeLimitFunctionalSuite) SetupSuite() {
-	s.FunctionalTestSuite.SetupSuiteWithCluster("testdata/sizelimit_cluster.yaml")
+	dynamicConfigOverrides := map[dynamicconfig.Key]any{
+		dynamicconfig.HistoryCountLimitWarn.Key():      10,
+		dynamicconfig.HistoryCountLimitError.Key():     20,
+		dynamicconfig.HistorySizeLimitWarn.Key():       5000,
+		dynamicconfig.HistorySizeLimitError.Key():      9000,
+		dynamicconfig.BlobSizeLimitWarn.Key():          1,
+		dynamicconfig.BlobSizeLimitError.Key():         1000,
+		dynamicconfig.MutableStateSizeLimitWarn.Key():  200,
+		dynamicconfig.MutableStateSizeLimitError.Key(): 1100,
+	}
+	s.FunctionalTestBase.SetupSuiteWithDefaultCluster(testcore.WithDynamicConfigOverrides(dynamicConfigOverrides))
 }
 
 func (s *SizeLimitFunctionalSuite) TestTerminateWorkflowCausedByHistoryCountLimit() {

tests/testcore/dynamic_config_overrides.go

@@ -0,0 +1,85 @@
+// The MIT License
+//
+// Copyright (c) 2020 Temporal Technologies Inc.  All rights reserved.
+//
+// Copyright (c) 2020 Uber Technologies, Inc.
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in
+// all copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+// THE SOFTWARE.
+
+package testcore
+
+import (
+	"time"
+
+	"go.temporal.io/server/common/dynamicconfig"
+	"go.temporal.io/server/common/persistence/visibility"
+)
+
+var (
+	// Functional tests don't use dynamic config files. All settings get their default values
+	// defined in common/dynamicconfig/constants.go.
+	//
+	// There are 4 ways to override a setting:
+	// 1. Globally using this file. Every test suite creates a new test cluster using this overrides.
+	// 2. Per test suite using FunctionalTestBase.SetupSuiteWithCluster() and WithDynamicConfigOverrides() option.
+	// 3. Per test using FunctionalTestBase.OverrideDynamicConfig() method.
+	// 4. Per specific cluster per test (if test has more than one cluster) using TestCluster.OverrideDynamicConfig() method.
+	//
+	// NOTE1: settings which are not really dynamic (requires server restart to take effect) can't be overridden on test level,
+	//        i.e., must be overridden globally (1) or per test suite (2).
+	// NOTE2: per test overrides change the value for the cluster, therefore, it affects not only a specific test, but
+	//        all tests for that suite. The automatic cleanup reverts to the  previous value and tests don't affect each other.
+	//        But that means tests in the same suite can't be run in parallel. This is not a problem because testify
+	//        doesn't allow parallel execution of tests in the same suite anyway. If one day, it will be allowed,
+	//        unique namespaces with overrides per namespace should be used for tests that require overrides.
+	dynamicConfigOverrides = map[dynamicconfig.Key]any{
+		dynamicconfig.FrontendRPS.Key():                                         3000,
+		dynamicconfig.FrontendMaxNamespaceVisibilityRPSPerInstance.Key():        50,
+		dynamicconfig.FrontendMaxNamespaceVisibilityBurstRatioPerInstance.Key(): 1,
+		dynamicconfig.ReplicationTaskProcessorErrorRetryMaxAttempts.Key():       1,
+		dynamicconfig.SecondaryVisibilityWritingMode.Key():                      visibility.SecondaryVisibilityWritingModeOff,
+		dynamicconfig.WorkflowTaskHeartbeatTimeout.Key():                        5 * time.Second,
+		dynamicconfig.ReplicationTaskFetcherAggregationInterval.Key():           200 * time.Millisecond,
+		dynamicconfig.ReplicationTaskFetcherErrorRetryWait.Key():                50 * time.Millisecond,
+		dynamicconfig.ReplicationTaskProcessorErrorRetryWait.Key():              time.Millisecond,
+		dynamicconfig.ClusterMetadataRefreshInterval.Key():                      100 * time.Millisecond,
+		dynamicconfig.NamespaceCacheRefreshInterval.Key():                       NamespaceCacheRefreshInterval,
+		dynamicconfig.ReplicationEnableUpdateWithNewTaskMerge.Key():             true,
+		dynamicconfig.ValidateUTF8SampleRPCRequest.Key():                        1.0,
+		dynamicconfig.ValidateUTF8SampleRPCResponse.Key():                       1.0,
+		dynamicconfig.ValidateUTF8SamplePersistence.Key():                       1.0,
+		dynamicconfig.ValidateUTF8FailRPCRequest.Key():                          true,
+		dynamicconfig.ValidateUTF8FailRPCResponse.Key():                         true,
+		dynamicconfig.ValidateUTF8FailPersistence.Key():                         true,
+		dynamicconfig.EnableWorkflowExecutionTimeoutTimer.Key():                 true,
+		dynamicconfig.FrontendMaskInternalErrorDetails.Key():                    false,
+		dynamicconfig.HistoryScannerEnabled.Key():                               false,
+		dynamicconfig.TaskQueueScannerEnabled.Key():                             false,
+		dynamicconfig.ExecutionsScannerEnabled.Key():                            false,
+		dynamicconfig.BuildIdScavengerEnabled.Key():                             false,
+
+		// Better to read through in tests than add artificial sleeps (which is what we previously had).
+		dynamicconfig.ForceSearchAttributesCacheRefreshOnRead.Key(): true,
+
+		dynamicconfig.RetentionTimerJitterDuration.Key():        time.Second,
+		dynamicconfig.EnableEagerWorkflowStart.Key():            true,
+		dynamicconfig.FrontendEnableExecuteMultiOperation.Key(): true,
+		dynamicconfig.ActivityAPIsEnabled.Key():                 true,
+	}
+)

tests/testcore/functional_test_base.go

@@ -53,6 +53,7 @@ import (
 	"go.temporal.io/server/common/namespace"
 	"go.temporal.io/server/common/payloads"
 	"go.temporal.io/server/common/persistence"
+	"go.temporal.io/server/common/persistence/visibility"
 	"go.temporal.io/server/common/primitives"
 	"go.temporal.io/server/common/primitives/timestamp"
 	"go.temporal.io/server/common/rpc"
@@ -207,23 +208,12 @@ func (s *FunctionalTestBase) SetupSuiteWithCluster(clusterConfigFile string, opt
 	s.Require().Empty(s.testClusterConfig.DeprecatedFrontendAddress, "Functional tests against external frontends are not supported")
 	s.Require().Empty(s.testClusterConfig.DeprecatedClusterNo, "ClusterNo should not be present in cluster config files")
 
-	if s.testClusterConfig.DynamicConfigOverrides == nil {
-		s.testClusterConfig.DynamicConfigOverrides = make(map[dynamicconfig.Key]any)
-	}
-
-	// TODO (alex): clusterConfig shouldn't have DC at all.
-	maps.Copy(s.testClusterConfig.DynamicConfigOverrides, map[dynamicconfig.Key]any{
-		dynamicconfig.HistoryScannerEnabled.Key():    false,
-		dynamicconfig.TaskQueueScannerEnabled.Key():  false,
-		dynamicconfig.ExecutionsScannerEnabled.Key(): false,
-		dynamicconfig.BuildIdScavengerEnabled.Key():  false,
-		// Better to read through in tests than add artificial sleeps (which is what we previously had).
-		dynamicconfig.ForceSearchAttributesCacheRefreshOnRead.Key(): true,
-		dynamicconfig.RetentionTimerJitterDuration.Key():            time.Second,
-		dynamicconfig.EnableEagerWorkflowStart.Key():                true,
-		dynamicconfig.FrontendEnableExecuteMultiOperation.Key():     true,
-	})
+	s.testClusterConfig.DynamicConfigOverrides = make(map[dynamicconfig.Key]any)
 	maps.Copy(s.testClusterConfig.DynamicConfigOverrides, params.DynamicConfigOverrides)
+	// TODO (alex): is it needed?
+	if s.testClusterConfig.ESConfig != nil {
+		s.testClusterConfig.DynamicConfigOverrides[dynamicconfig.SecondaryVisibilityWritingMode.Key()] = visibility.SecondaryVisibilityWritingModeDual
+	}
 
 	s.testClusterConfig.ServiceFxOptions = params.ServiceOptions
 	s.testClusterConfig.EnableMetricsCapture = true
@@ -259,15 +249,12 @@ func (s *FunctionalTestBase) SetupSuiteWithCluster(clusterConfigFile string, opt
 	s.adminClient = s.testCluster.AdminClient()
 	s.operatorClient = s.testCluster.OperatorClient()
 	s.httpAPIAddress = s.testCluster.Host().FrontendHTTPAddress()
-
 }
 
 // All test suites that inherit FunctionalTestBase and overwrite SetupTest must
 // call this testcore FunctionalTestBase.SetupTest function to distribute the tests
 // into partitions. Otherwise, the test suite will be executed multiple times
 // in each partition.
-// Furthermore, all test suites in the "tests/" directory that don't inherit
-// from FunctionalTestBase must implement SetupTest that calls checkTestShard.
 func (s *FunctionalTestBase) SetupTest() {
 	s.checkTestShard()
 	s.initAssertions()